diff --git a/dns/internal.yaml b/dns/internal.yaml index 31ed52b..e840141 100644 --- a/dns/internal.yaml +++ b/dns/internal.yaml @@ -127,4 +127,5 @@ aliases: - {name: wherethef, site: nh3, target: nh3-dev, note: WhereTF :8093} - {name: homepage, site: esh, target: esh-docker-vm, note: fleet dashboard :5100} - {name: scriberr, site: fv, target: fv-ml1, note: transcription + diarization :8080 (GPU1)} - - {name: semif, site: fv, target: fv-ml1, note: SemIf option-logit decisions :8032 (GPU1)} + - {name: semif, site: fv, target: fv-ml1, note: SemIf option-logit decisions :8032 (GPU1) — stopped since 2026-09-30, being replaced by intern-decision (Prime); kept as rollback} + - {name: intern-decision, site: fv, target: fv-ml1, note: Intern-Decision-4B typed decisions :8033 (GPU1), semif-serve API; replaces semif (Prime 2026-09-30)} diff --git a/services/intern-decision-serve/acceptance/checks.py b/services/intern-decision-serve/acceptance/checks.py new file mode 100644 index 0000000..0b7c0fd --- /dev/null +++ b/services/intern-decision-serve/acceptance/checks.py @@ -0,0 +1,243 @@ +"""intern-decision-serve acceptance: the service-level checks the bench harness does not cover. +Stdlib only, so it runs on fv-ml1's host python or on nh3-dev against a live instance. + + python3 checks.py --url URL --token-file T --out out.json --checks auth,chunking,queue,maxreq + python3 checks.py --url URL --token-file T --out oom.json --checks oom + + auth 401 without / with a wrong token on both POSTs; /health open + chunking 20 decisions in one /decide/shared equal the same decisions asked as 16 + 4 + queue past MAX_QUEUE requests in progress, the rest get 429 busy (and the admitted all answer) + maxreq the largest request the API accepts: MAX_DECISIONS decisions x 16 options, the state + grown until each 16-question call sits just under MAX_TOKENS (found from the model's own + 422 message). Sent --repeats times; status, tokens and /health's allocator peak recorded. + fit under the cap, the largest 16-question call that answers 200 (binary search, in tokens) + oom the over-cap request (maxreq's body when it ran) -> 503 out_of_memory, then reserved memory + back to its resting value and an ordinary request answered 200. Earlier text: a long request against a deliberately tight cap -> 503 out_of_memory, then reserved + memory back to its resting value and an ordinary request answered 200. +Every request's status, timing and the fields that prove the point are kept in --out. +""" +from __future__ import annotations + +import argparse +import json +import re +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +YESNO = [{"id": "yes", "description": "Yes"}, {"id": "no", "description": "No"}] + + +def post(url, body, headers, timeout=600): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", **headers}) + t = time.perf_counter() + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + status, raw = r.status, r.read() + except urllib.error.HTTPError as e: + status, raw = e.code, e.read() + ms = (time.perf_counter() - t) * 1000 + try: + return status, json.loads(raw), ms + except ValueError: + return status, {"raw": raw[:300].decode(errors="replace")}, ms + + +def get(url): + with urllib.request.urlopen(url, timeout=30) as r: + return r.status, json.loads(r.read()) + + +def criteria(n, options=YESNO, prefix="c"): + return [{"id": f"{prefix}{i}", "question": f"Does the evidence mention item number {i}?", "options": options} + for i in range(n)] + + +def check_auth(url, h): + out = {} + body = {"id": "a", "state": "s", "question": "q?", "options": YESNO} + for path, b in (("/decide", body), ("/decide/shared", {"state": "s", "decisions": [body]})): + out[path] = {"no_token": post(url + path, b, {})[0], + "wrong_token": post(url + path, b, {"Authorization": "Bearer " + "x" * 48})[0], + "right_token": post(url + path, b, h)[0]} + out["health_no_token"] = get(url + "/health")[0] + out["pass"] = all(v["no_token"] == 401 and v["wrong_token"] == 401 and v["right_token"] == 200 + for k, v in out.items() if k.startswith("/")) and out["health_no_token"] == 200 + return out + + +def check_chunking(url, h, state): + crits = [{"id": f"d{i}", "question": q, "options": opts} for i, (q, opts) in enumerate([ + (f"Is item {i} a {kind}?", [{"id": "yes", "description": "Yes"}, {"id": "no", "description": "No"}, + {"id": "unclear", "description": "Cannot tell from the evidence"}]) + for i, kind in enumerate(["policy", "exception", "deadline", "person", "number"] * 4)])] + s_all, j_all, ms_all = post(url + "/decide/shared", {"state": state, "decisions": crits}, h) + s_a, j_a, _ = post(url + "/decide/shared", {"state": state, "decisions": crits[:16]}, h) + s_b, j_b, _ = post(url + "/decide/shared", {"state": state, "decisions": crits[16:]}, h) + split = (j_a.get("results", []) + j_b.get("results", [])) if s_a == s_b == 200 else [] + rows = [] + for whole, part in zip(j_all.get("results", []), split): + rows.append({"id": whole["id"], "top_whole": whole["top"], "top_split": part["top"], + "max_dp": max(abs(x - y) for x, y in zip(whole["probabilities"], part["probabilities"])), + "call_whole": whole["call"], "sha_equal": whole["prompt_sha256"] == part["prompt_sha256"]}) + t = j_all.get("timing", {}) + return {"status": [s_all, s_a, s_b], "timing": t, "rows": rows, "e2e_ms": round(ms_all, 1), + "pass": s_all == s_a == s_b == 200 and t.get("calls") == 2 and t.get("questions_per_call") == [16, 4] + and len(rows) == 20 and all(r["top_whole"] == r["top_split"] and r["max_dp"] == 0 and r["sha_equal"] + for r in rows)} + + +def check_queue(url, h, state, max_queue, extra): + n = max_queue + extra + body = {"state": state, "decisions": criteria(16)} + barrier = threading.Barrier(n) + results = [None] * n + + def one(i): + barrier.wait() + s, j, ms = post(url + "/decide/shared", body, h) + results[i] = (s, (j.get("error") or {}).get("code"), round(ms, 1)) + + threads = [threading.Thread(target=one, args=(i,)) for i in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + statuses = [r[0] for r in results] + return {"sent": n, "max_queue": max_queue, "ok": statuses.count(200), "busy_429": statuses.count(429), + "codes_429": sorted({r[1] for r in results if r[0] == 429}), "other": [r for r in results if r[0] not in (200, 429)], + "pass": statuses.count(429) >= 1 and statuses.count(200) + statuses.count(429) == n + and statuses.count(200) <= max_queue} + + +def question_block(i): + """Zero-padded numbers: Qwen tokenises digits one by one, so every 16-question call of a + 64-decision request has the same token count and each one sits at the limit.""" + return {"id": f"q{i:02d}", "question": f"Which of the listed categories best describes clause {i:02d} of the policy?", + "options": [{"id": f"k{j:02d}", "description": f"Category {j:02d}: the clause concerns topic number {j:02d}"} + for j in range(16)]} + + +def find_max_state(url, h, base_state, max_tokens, n_decisions=16): + """Grow the state (repeating base_state) until the request is refused with 422, using the token + count the model's own 422 reports; return the longest state the API accepts and its tokens per call. + Sized with the WHOLE n_decisions request, so the largest call in it is the one that decides.""" + decisions = [question_block(i) for i in range(n_decisions)] + text = (base_state + "\n\n") * 4 + + def tokens_for(chars): + s, j, _ = post(url + "/decide/shared", {"state": text[:chars], "decisions": decisions}, h) + if s == 200: + return s, j["timing"]["input_tokens"][0] + if s == 503: + return s, None + m = re.search(r"has (\d+) tokens", (j.get("error") or {}).get("message", "")) + return s, int(m.group(1)) if m else None + + lo, hi = 1000, len(text) + s, tok = tokens_for(hi) + assert s == 422 and tok and tok > max_tokens, (s, tok) + best = None + while lo < hi: + mid = (lo + hi + 1) // 2 + s, tok = tokens_for(mid) + if s in (200, 503): # accepted by the API (503 = accepted, then refused by the VRAM cap) + lo, best = mid, tok + elif s == 422: + hi = mid - 1 + else: + raise RuntimeError(f"unexpected {s} while sizing the max request: {tok}") + return text[:lo], best + + +MAX_BODY = {} + + +def check_maxreq(url, h, base_state, max_tokens, max_decisions, repeats): + state, tokens = find_max_state(url, h, base_state, max_tokens, max_decisions) + decisions = [question_block(i) for i in range(max_decisions)] + body = {"state": state, "decisions": decisions} + MAX_BODY.update(body) + size = len(json.dumps(body).encode()) + runs = [] + for _ in range(repeats): + s, j, ms = post(url + "/decide/shared", body, h) + health = get(url + "/health")[1]["model"] + runs.append({"status": s, "e2e_ms": round(ms, 1), "code": (j.get("error") or {}).get("code"), + "calls": (j.get("timing") or {}).get("calls"), "input_tokens": (j.get("timing") or {}).get("input_tokens"), + "reserved_gib_after": health.get("reserved_gib"), "max_reserved_gib": health.get("max_reserved_gib"), + "t_end": time.time()}) + return {"state_chars": len(state), "tokens_per_call": tokens, "decisions": max_decisions, "options_each": 16, + "body_bytes": size, "runs": runs} + + +def check_fit(url, h, base_state): + """Under the cap: the largest 16-question call (16 options each) that still answers 200, in tokens.""" + decisions = [question_block(i) for i in range(16)] + text = (base_state + "\n\n") * 4 + lo, hi, best, probes = 1000, len(text), None, [] + while lo < hi: + mid = (lo + hi + 1) // 2 + s, j, _ = post(url + "/decide/shared", {"state": text[:mid], "decisions": decisions}, h) + probes.append([mid, s, j["timing"]["input_tokens"][0] if s == 200 else None]) + if s == 200: + lo, best = mid, j["timing"]["input_tokens"][0] + else: + hi = mid - 1 + return {"largest_ok_state_chars": lo, "largest_ok_tokens": best, "probes": probes} + + +def check_oom(url, h, long_state): + """The over-cap request: the largest one (maxreq) if it ran, else 16 criteria over the long state.""" + before = get(url + "/health")[1]["model"] + body = dict(MAX_BODY) or {"state": long_state, "decisions": criteria(16)} + s, j, ms = post(url + "/decide/shared", body, h) + after = get(url + "/health")[1]["model"] + s2, j2, _ = post(url + "/decide", {"id": "after", "state": "The deploy passed.", "question": "Did it pass?", + "options": YESNO}, h) + again = get(url + "/health")[1]["model"] + return {"oom_status": s, "oom_error": j.get("error"), "oom_ms": round(ms, 1), + "reserved_gib": {"before": before.get("reserved_gib"), "after_oom": after.get("reserved_gib"), + "after_next_request": again.get("reserved_gib")}, + "next_request_status": s2, "next_top": j2.get("top"), + "pass": s == 503 and (j.get("error") or {}).get("code") == "out_of_memory" and s2 == 200 + and after.get("reserved_gib") is not None and after["reserved_gib"] <= before["reserved_gib"] + 0.5} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--url", required=True) + ap.add_argument("--token-file", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--checks", default="auth,chunking,queue,maxreq") + ap.add_argument("--long-state-file", default=str(HERE / "long_state.txt")) + ap.add_argument("--max-queue", type=int, default=32) + ap.add_argument("--queue-extra", type=int, default=16) + ap.add_argument("--repeats", type=int, default=3) + args = ap.parse_args() + url = args.url.rstrip("/") + h = {"Authorization": "Bearer " + Path(args.token_file).read_text().strip()} + long_state = Path(args.long_state_file).read_text() + health = get(url + "/health")[1] + report = {"url": url, "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "health": health} + for c in args.checks.split(","): + t = time.time() + report[c] = {"auth": lambda: check_auth(url, h), + "chunking": lambda: check_chunking(url, h, long_state[:4000]), + "queue": lambda: check_queue(url, h, long_state, args.max_queue, args.queue_extra), + "maxreq": lambda: check_maxreq(url, h, long_state, health["max_tokens"], health["max_decisions"], + args.repeats), + "fit": lambda: check_fit(url, h, long_state), + "oom": lambda: check_oom(url, h, long_state)}[c]() + report[c]["t_start"], report[c]["t_end"] = t, time.time() + print(c, json.dumps({k: v for k, v in report[c].items() if k not in ("rows", "runs")})[:600], flush=True) + report["finished_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + Path(args.out).write_text(json.dumps(report, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/services/intern-decision-serve/acceptance/compare.py b/services/intern-decision-serve/acceptance/compare.py new file mode 100644 index 0000000..8c9b11c --- /dev/null +++ b/services/intern-decision-serve/acceptance/compare.py @@ -0,0 +1,101 @@ +"""Score intern-decision-serve acceptance runs of the 2026-09-30 bench harness (bench_sets.py +--backend semif, i.e. through the service's semif-compatible API) exactly as the bench's analyze.py +scored them, and compare them row by row with the bench's own native Intern-Decision rows. + + python3 compare.py --ours r1/sets.json r2/sets.json r3/sets.json \ + --bench ../../semif-serve/bench-jev-2026-09-30/raw/out/intern-decision-4b-native > compare.json + +Definitions (from analyze.py): a row counts when ok and labelled; right = top in gold. pooled = +authored144 + cicada-w1 + wyrd evidence rows (259). The negative control is read against the same +run's single condition. +""" +from __future__ import annotations + +import argparse +import json +import statistics as st +from pathlib import Path + +SETS = ("authored144", "perturbations108", "cicada-w1", "cicada-w2", "wyrd") +POOLED = ("authored144", "cicada-w1", "wyrd") +NAMES = ("pooled",) + SETS + ("wyrd:place", "wyrd:place2", "wyrd:exit", "wyrd:exit2") + + +def rows(run: dict, cond: str) -> dict: + c = run["conditions"].get(cond) + return {x["id"]: x for x in c["rows"]} if c else {} + + +def acc(rs) -> tuple[int, int]: + lab = [x for x in rs if x["ok"] and x["gold"]] + return sum(x["top"] in x["gold"] for x in lab), len(lab) + + +def set_scores(run: dict) -> dict: + out = {} + for cond in ("single", "rotations", "multifield"): + rr = list(rows(run, cond).values()) + if not rr: + continue + for name in NAMES: + base, _, dec = name.partition(":") + wanted = set(POOLED) if base == "pooled" else {base} + sub = [x for x in rr if x["set"] in wanted and (not dec or x["decision"] == dec) and x["cond"] == "evidence"] + c, n = acc(sub) + if n: + out[f"{cond}/{name}"] = [c, n] + out[f"{cond}/failures"] = sum(not x["ok"] for x in rr) + return out + + +def negative(run: dict) -> dict: + s, neg = rows(run, "single"), rows(run, "negative") + ids = [i for i in neg if i in s and s[i]["ok"] and neg[i]["ok"]] + return {"n": len(ids), "same_top_as_unrotated": sum(neg[i]["top"] == s[i]["top"] for i in ids), + "follows_description": sum(neg[i]["top"] == neg[i]["gold_desc_id"] for i in ids), + "vs_original_gold": sum(neg[i]["top"] in neg[i]["gold"] for i in ids)} + + +def pdiff(a: dict, b: dict) -> float: + return max(abs(x - y) for x, y in zip(a["probs"], b["probs"])) + + +def pair(a: dict, b: dict, cond_a: str, cond_b: str) -> dict: + ra, rb = rows(a, cond_a), rows(b, cond_b) + ids = [i for i in ra if i in rb and ra[i]["ok"] and rb[i]["ok"]] + flips = [i for i in ids if ra[i]["top"] != rb[i]["top"]] + return {"rows": len(ids), "top_differs": len(flips), "flipped_ids": flips[:20], + "max_dp": max((pdiff(ra[i], rb[i]) for i in ids), default=None)} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--ours", nargs="+", required=True) + ap.add_argument("--bench", required=True, help="raw/out/intern-decision-4b-native") + args = ap.parse_args() + ours = {Path(p).parent.name or p: json.loads(Path(p).read_text()) for p in args.ours} + bench = {d.name: json.loads((d / "sets.json").read_text()) for d in sorted(Path(args.bench).glob("r*")) + if (d / "sets.json").exists()} + report = {"ours": {k: {"scores": set_scores(v), "negative": negative(v), + "a_vs_a_in_process_authored144": pair(v, v, "single", "repeat")} for k, v in ours.items()}, + "bench": {k: {"scores": set_scores(v), "negative": negative(v)} for k, v in bench.items()}} + def bench_with(cond: str) -> str: + """The first bench repeat that ran `cond` (multifield was added after the bench's r1).""" + return next(k for k in sorted(bench) if cond in bench[k]["conditions"]) + report["vs_bench_row_by_row"] = {k: {cond: {"bench_repeat": bench_with(cond), **pair(v, bench[bench_with(cond)], cond, cond)} + for cond in ("single", "rotations", "negative", "multifield")} + for k, v in ours.items()} + names = sorted(ours) + report["across_restarts"] = {f"{a}~{b}": {cond: pair(ours[a], ours[b], cond, cond) for cond in ("single", "rotations")} + for i, a in enumerate(names) for b in names[i + 1:]} + keys = sorted({k for v in report["ours"].values() for k in v["scores"] if not k.endswith("/failures")}) + report["summary"] = {k: {"ours_median": st.median(v["scores"][k][0] for v in report["ours"].values() if k in v["scores"]), + "ours_all": [v["scores"][k][0] for v in report["ours"].values() if k in v["scores"]], + "bench_all": [v["scores"][k][0] for v in report["bench"].values() if k in v["scores"]], + "n": next(v["scores"][k][1] for v in report["ours"].values() if k in v["scores"])} + for k in keys} + print(json.dumps(report, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap8.5-startup-refused/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap8.5-startup-refused/vram.csv new file mode 100644 index 0000000..d23d79b --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap8.5-startup-refused/vram.csv @@ -0,0 +1 @@ +# pid=0 gpu= diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/checks.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/checks.json new file mode 100644 index 0000000..20ccd43 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/checks.json @@ -0,0 +1,142 @@ +{ + "url": "http://127.0.0.1:18033", + "started_utc": "2026-09-30T16:36:04Z", + "health": { + "status": "ok", + "model": { + "name": "Intern-Decision-4B", + "source": "internlm/Intern-Decision-4B", + "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "checkpoint": "/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "inference_py_sha256": "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863", + "temperature": 1.99241824, + "dtype": "bfloat16", + "attn_implementation": "sdpa", + "device": "cuda", + "max_length": 8192, + "torch_version": "2.10.0+cu128", + "transformers_version": "5.17.0", + "vision_tower": "removed", + "device_name": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", + "allocated_gib": 7.937, + "reserved_gib": 7.969, + "max_reserved_gib": 8.861 + }, + "vram_cap_gib": 9.0, + "max_tokens": 8192, + "max_decisions": 64, + "max_questions_per_call": 16, + "chunking": "/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.", + "workloads": [] + }, + "fit": { + "largest_ok_state_chars": 4793, + "largest_ok_tokens": 7168, + "probes": [ + [ + 31950, + 422, + null + ], + [ + 16475, + 422, + null + ], + [ + 8737, + 503, + null + ], + [ + 4868, + 503, + null + ], + [ + 2934, + 200, + 6772 + ], + [ + 3901, + 200, + 6964 + ], + [ + 4384, + 200, + 7056 + ], + [ + 4626, + 200, + 7133 + ], + [ + 4747, + 200, + 7156 + ], + [ + 4807, + 503, + null + ], + [ + 4777, + 200, + 7163 + ], + [ + 4792, + 200, + 7168 + ], + [ + 4799, + 503, + null + ], + [ + 4795, + 503, + null + ], + [ + 4793, + 200, + 7168 + ], + [ + 4794, + 503, + null + ] + ], + "t_start": 1790786164.5822356, + "t_end": 1790786175.4811816 + }, + "maxreq": { + "state_chars": 9508, + "tokens_per_call": null, + "decisions": 64, + "options_each": 16, + "body_bytes": 101019, + "runs": [ + { + "status": 503, + "e2e_ms": 209.5, + "code": "out_of_memory", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.0, + "t_end": 1790786178.99772 + } + ], + "t_start": 1790786175.481284, + "t_end": 1790786178.9977298 + }, + "finished_utc": "2026-09-30T16:36:18Z" +} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/shape.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/shape.json new file mode 100644 index 0000000..b11bd1a --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/shape.json @@ -0,0 +1 @@ +{"label": "cap9.0", "backend": "semif", "runs": 1, "per": 5, "max_questions": 10000, "events": [["start", 1790786179.0584767], ["short1:measure", 1790786179.1493392], ["crit21:measure", 1790786179.5161796], ["long1:measure", 1790786186.5989966], ["long16:measure", 1790786194.1815286], ["shapes:done", 1790786195.251973], ["end", 1790786195.251976]], "shapes": {"short1": {"rows": 1, "runs": [{"e2e_ms": [37.941707065328956, 38.23350998573005, 37.37540193833411, 38.53617515414953, 38.11007016338408], "server_ms": [35.29082587920129, 35.39266809821129, 34.68621987849474, 35.76885210350156, 35.205855034291744], "statuses": [200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 38.11007016338408, "median_server": 35.29082587920129}], "tokens": 181, "median_e2e_ms": 38.1, "run_medians_e2e_ms": [38.1], "median_server_ms": 35.3, "non_200": 0}, "crit21": {"rows": 21, "runs": [{"e2e_ms": [86.08735189773142, 85.76549892313778, 85.83429013378918, 85.84657986648381, 85.68961801938713], "server_ms": [79.76265903562307, 79.48806718923151, 79.50853602960706, 79.57211695611477, 79.62968898937106], "statuses": [200, 200, 200, 200, 200], "tokens": null, "median_e2e": 85.83429013378918, "median_server": 79.57211695611477}], "tokens": null, "median_e2e_ms": 85.8, "run_medians_e2e_ms": [85.8], "median_server_ms": 79.6, "non_200": 0}, "long1": {"rows": 1, "runs": [{"e2e_ms": [184.56048821099102, 191.5509181562811, 187.36126995645463, 186.49141117930412, 187.67126300372183], "server_ms": [181.11044797115028, 187.99379700794816, 183.76171891577542, 183.00863006152213, 184.1394929215312], "statuses": [200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 187.36126995645463, "median_server": 183.76171891577542}], "tokens": 4019, "median_e2e_ms": 187.4, "run_medians_e2e_ms": [187.4], "median_server_ms": 183.8, "non_200": 0}, "long16": {"rows": 16, "runs": [{"e2e_ms": [211.36997686699033, 217.9243720602244, 213.36712990887463, 213.6664930731058, 211.49204880930483], "server_ms": [205.53427003324032, 212.03652513213456, 207.65956514514983, 207.81763619743288, 205.71005111560225], "statuses": [200, 200, 200, 200, 200], "tokens": null, "median_e2e": 213.36712990887463, "median_server": 207.65956514514983}], "tokens": null, "median_e2e_ms": 213.4, "run_medians_e2e_ms": [213.4], "median_server_ms": 207.7, "non_200": 0}}} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/vram.csv new file mode 100644 index 0000000..a79e9c3 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/cap9.0/vram.csv @@ -0,0 +1,294 @@ +# pid=1899253 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790786164.716320980,8812,8822 +1790786164.823793359,8812,8822 +1790786164.929302716,8812,8822 +1790786165.035433570,8812,9408 +1790786165.148395222,9398,9664 +1790786165.259687015,9654,9664 +1790786165.372175832,9654,9664 +1790786165.484222484,9654,9664 +1790786165.590480950,9654,9664 +1790786165.709071847,9654,9664 +1790786165.813055136,9654,9664 +1790786165.915808311,9654,9664 +1790786166.018890279,9654,9664 +1790786166.129017919,9654,9664 +1790786166.229606519,9654,9664 +1790786166.335587621,9654,9664 +1790786166.443510165,9654,9664 +1790786166.547975000,9654,9664 +1790786166.655635241,9654,9664 +1790786166.761522382,9654,9664 +1790786166.867852718,9654,9664 +1790786166.977569563,9654,9664 +1790786167.085987964,9654,9664 +1790786167.196082253,9654,9664 +1790786167.305719237,9654,9664 +1790786167.416458704,9654,9664 +1790786167.523194095,9654,9664 +1790786167.632373654,9654,9664 +1790786167.731704019,9654,9664 +1790786167.840186040,9654,9664 +1790786167.944049348,9654,9664 +1790786168.044211153,9654,9664 +1790786168.145394139,9654,9664 +1790786168.253804889,9654,9664 +1790786168.356489443,9654,9664 +1790786168.471620971,9654,9664 +1790786168.569846013,9654,9664 +1790786168.683211431,9654,9664 +1790786168.785017705,9654,9664 +1790786168.896854124,9654,9664 +1790786169.000832373,9654,9664 +1790786169.117628050,9654,9664 +1790786169.217906256,9654,9664 +1790786169.322926357,9654,9664 +1790786169.431459258,9654,9664 +1790786169.537479991,9654,9664 +1790786169.645614758,9654,9664 +1790786169.755306653,9654,9664 +1790786169.861792561,9654,9664 +1790786169.981926806,9654,9664 +1790786170.081930029,9654,9664 +1790786170.197530461,9654,9664 +1790786170.301479580,9654,9664 +1790786170.417932963,9654,9664 +1790786170.519371663,9654,9664 +1790786170.630502444,9654,9664 +1790786170.737537438,9654,9664 +1790786170.845572454,9654,9664 +1790786170.948239538,9654,9664 +1790786171.060100188,9654,9664 +1790786171.168635179,9654,9664 +1790786171.272443606,9654,9664 +1790786171.379895525,9654,9664 +1790786171.483702352,9846,9842 +1790786171.585724958,9832,9842 +1790786171.688623185,9832,8824 +1790786171.792108018,9828,9838 +1790786171.893702910,9828,8824 +1790786172.000577552,9848,9858 +1790786172.107827978,9848,9858 +1790786172.210922127,9848,8824 +1790786172.320767474,9852,9866 +1790786172.427548455,9856,9866 +1790786172.535807333,9856,9866 +1790786172.641415401,9584,9876 +1790786172.749858882,9866,9876 +1790786172.857379212,9866,9876 +1790786172.961890846,8814,9876 +1790786173.068642918,9866,9876 +1790786173.177382221,9866,9876 +1790786173.283539615,9866,9370 +1790786173.384837343,9866,9876 +1790786173.489906035,9866,9876 +1790786173.596368152,9866,8826 +1790786173.696510917,9828,9838 +1790786173.800309834,9828,9444 +1790786173.908233539,9586,9876 +1790786174.017859623,9866,9876 +1790786174.124053877,9866,9876 +1790786174.229679985,8814,9876 +1790786174.333861606,9838,9848 +1790786174.437805694,9838,9848 +1790786174.543411892,9838,9702 +1790786174.642070150,9828,9838 +1790786174.740548365,9828,8824 +1790786174.839833831,9828,9838 +1790786174.937788440,9828,8824 +1790786175.042638529,9866,9848 +1790786175.147763531,9838,9848 +1790786175.251046852,9838,9848 +1790786175.357220336,9272,9838 +1790786175.462097365,9828,9838 +1790786175.563701577,8814,8824 +1790786175.658918195,8814,8824 +1790786175.756543930,8814,8824 +1790786175.857112750,8814,8824 +1790786175.955723127,8814,8824 +1790786176.057100796,8814,8824 +1790786176.164637056,9016,9856 +1790786176.269457695,9846,9856 +1790786176.369247326,8814,8824 +1790786176.468768483,8814,8824 +1790786176.572184595,8814,8824 +1790786176.680519425,8814,8824 +1790786176.787843012,8814,8824 +1790786176.895493653,8814,9864 +1790786177.002038592,9854,9864 +1790786177.107059683,9110,9480 +1790786177.206975745,9854,9864 +1790786177.308243823,9854,8824 +1790786177.407296535,8814,8824 +1790786177.511630138,8814,8824 +1790786177.613017667,8814,8824 +1790786177.717066116,9274,9864 +1790786177.815656494,9854,9864 +1790786177.915900859,8814,9864 +1790786178.018981288,9854,9864 +1790786178.117579505,8814,8824 +1790786178.217936202,8814,8824 +1790786178.319862747,8814,8824 +1790786178.419795330,8814,8824 +1790786178.525969424,9854,9864 +1790786178.628061531,9854,8824 +1790786178.728005853,8814,8824 +1790786178.833653081,8814,9736 +1790786178.937844083,9854,9864 +1790786179.037766255,9854,8824 +1790786179.140034804,8820,8830 +1790786179.245360839,8820,8830 +1790786179.349544090,8820,8830 +1790786179.454476600,8940,8950 +1790786179.557948563,8940,8950 +1790786179.660475705,8940,8950 +1790786179.760389278,8940,8950 +1790786179.866003435,8940,8950 +1790786179.966888758,8940,8950 +1790786180.078586336,9148,9414 +1790786180.193379950,9404,9414 +1790786180.301062822,9404,9414 +1790786180.409891267,9404,9414 +1790786180.511185555,9404,9414 +1790786180.615584188,9404,9414 +1790786180.722617943,9404,9414 +1790786180.825872873,9404,9414 +1790786180.927496455,9404,9414 +1790786181.036240609,9404,9414 +1790786181.139894494,9404,9414 +1790786181.239871237,9404,9414 +1790786181.341871773,9404,9414 +1790786181.444553357,9404,9414 +1790786181.545995006,9404,9414 +1790786181.645816877,9404,9414 +1790786181.747032645,9404,9414 +1790786181.858655222,9404,9414 +1790786181.954165593,9404,9414 +1790786182.068152478,9404,9414 +1790786182.179624703,9404,9414 +1790786182.293777879,9404,9414 +1790786182.394289098,9404,9414 +1790786182.505242597,9404,9414 +1790786182.608015352,9404,9414 +1790786182.711196992,9404,9414 +1790786182.819210728,9404,9414 +1790786182.922884623,9404,9414 +1790786183.022149047,9404,9414 +1790786183.127445742,9404,9414 +1790786183.231498132,9404,9414 +1790786183.352851121,9404,9414 +1790786183.455674986,9404,9414 +1790786183.569989714,9404,9414 +1790786183.673774791,9404,9414 +1790786183.788232161,9404,9414 +1790786183.893344203,9404,9414 +1790786184.000703941,9404,9414 +1790786184.102348013,9404,9414 +1790786184.203479409,9404,9414 +1790786184.312577357,9404,9414 +1790786184.423768159,9404,9414 +1790786184.525157428,9404,9414 +1790786184.638123101,9404,9414 +1790786184.736714128,9404,9414 +1790786184.849963804,9404,9414 +1790786184.956039516,9404,9414 +1790786185.061942268,9404,9414 +1790786185.160747317,9404,9414 +1790786185.273083292,9404,9414 +1790786185.375250500,9404,9414 +1790786185.480599585,9404,9414 +1790786185.585149950,9404,9414 +1790786185.691940402,9404,9414 +1790786185.800535864,9404,9414 +1790786185.901016463,9404,9414 +1790786186.017956101,9404,9414 +1790786186.119091647,9404,9414 +1790786186.230213799,9404,9414 +1790786186.335299000,9404,9414 +1790786186.440112419,9404,9624 +1790786186.542811883,9614,9624 +1790786186.644845759,9370,9624 +1790786186.752999377,9614,9624 +1790786186.861355316,8878,9624 +1790786186.966925093,9614,9624 +1790786187.073369551,9614,9624 +1790786187.178632085,9614,8824 +1790786187.281926276,9614,9624 +1790786187.394704916,9614,9624 +1790786187.504238909,9614,9624 +1790786187.613575700,8840,9160 +1790786187.727728016,9406,9416 +1790786187.832223631,9406,9416 +1790786187.941769684,9406,9416 +1790786188.047064828,9406,9416 +1790786188.144852405,9406,9416 +1790786188.260695991,9406,9416 +1790786188.363770900,9406,9416 +1790786188.468639289,9406,9416 +1790786188.572878751,9406,9416 +1790786188.681629185,9406,9416 +1790786188.785445422,9406,9416 +1790786188.885867330,9406,9416 +1790786188.990390325,9406,9416 +1790786189.091682043,9406,9416 +1790786189.190010677,9406,9416 +1790786189.292761142,9406,9416 +1790786189.399216419,9406,9416 +1790786189.506859511,9406,9416 +1790786189.617929631,9406,9416 +1790786189.720011158,9406,9416 +1790786189.830994678,9406,9416 +1790786189.932984344,9406,9416 +1790786190.042418256,9406,9416 +1790786190.142131905,9406,9416 +1790786190.243875889,9406,9416 +1790786190.357939294,9406,9416 +1790786190.459039669,9406,9416 +1790786190.559216775,9406,9416 +1790786190.664521479,9406,9416 +1790786190.773597677,9406,9416 +1790786190.884632127,9406,9416 +1790786190.991845523,9406,9416 +1790786191.108368816,9406,9416 +1790786191.210100890,9406,9416 +1790786191.320561794,9406,9416 +1790786191.422654041,9406,9416 +1790786191.537225172,9406,9416 +1790786191.641148860,9406,9416 +1790786191.745807307,9406,9416 +1790786191.861669633,9406,9416 +1790786191.968884069,9406,9416 +1790786192.071584373,9406,9416 +1790786192.185177483,9406,9416 +1790786192.287305381,9406,9416 +1790786192.397678813,9406,9416 +1790786192.497060839,9406,9416 +1790786192.604761661,9406,9416 +1790786192.704385470,9406,9416 +1790786192.817246931,9406,9416 +1790786192.918724221,9406,9416 +1790786193.024778954,9406,9416 +1790786193.128297328,9406,9416 +1790786193.236105151,9406,9416 +1790786193.340450404,9406,9416 +1790786193.440592759,9406,9416 +1790786193.547516322,9406,9416 +1790786193.648185103,9406,9416 +1790786193.764486983,9406,9416 +1790786193.864659308,9480,9490 +1790786193.968057501,9480,9490 +1790786194.070461872,9722,9732 +1790786194.176537044,9722,9732 +1790786194.280394772,9722,9732 +1790786194.387266254,9722,9732 +1790786194.493886684,9722,9732 +1790786194.597803502,9722,9732 +1790786194.698275231,9722,9732 +1790786194.807813474,9722,9732 +1790786194.913704905,9722,9732 +1790786195.017218788,9722,9732 +1790786195.129467792,9722,9732 +1790786195.239846675,9722,9732 +1790786195.357501522,8814,8824 +1790786195.466641560,8814,8824 +1790786195.629528039,0,2 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/compare.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/compare.json new file mode 100644 index 0000000..eaad744 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/compare.json @@ -0,0 +1,1485 @@ +{ + "ours": { + "r1": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0, + "multifield/pooled": [ + 78, + 84 + ], + "multifield/wyrd": [ + 78, + 84 + ], + "multifield/wyrd:place": [ + 17, + 21 + ], + "multifield/wyrd:place2": [ + 21, + 21 + ], + "multifield/wyrd:exit": [ + 20, + 21 + ], + "multifield/wyrd:exit2": [ + 20, + 21 + ], + "multifield/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + }, + "a_vs_a_in_process_authored144": { + "rows": 144, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + } + }, + "r2": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0, + "multifield/pooled": [ + 78, + 84 + ], + "multifield/wyrd": [ + 78, + 84 + ], + "multifield/wyrd:place": [ + 17, + 21 + ], + "multifield/wyrd:place2": [ + 21, + 21 + ], + "multifield/wyrd:exit": [ + 20, + 21 + ], + "multifield/wyrd:exit2": [ + 20, + 21 + ], + "multifield/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + }, + "a_vs_a_in_process_authored144": { + "rows": 144, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + } + }, + "r3": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0, + "multifield/pooled": [ + 78, + 84 + ], + "multifield/wyrd": [ + 78, + 84 + ], + "multifield/wyrd:place": [ + 17, + 21 + ], + "multifield/wyrd:place2": [ + 21, + 21 + ], + "multifield/wyrd:exit": [ + 20, + 21 + ], + "multifield/wyrd:exit2": [ + 20, + 21 + ], + "multifield/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + }, + "a_vs_a_in_process_authored144": { + "rows": 144, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + } + }, + "oom": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0 + }, + "negative": { + "n": 0, + "same_top_as_unrotated": 0, + "follows_description": 0, + "vs_original_gold": 0 + }, + "a_vs_a_in_process_authored144": { + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + } + } + }, + "bench": { + "r1": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + } + }, + "r2": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0, + "multifield/pooled": [ + 77, + 84 + ], + "multifield/wyrd": [ + 77, + 84 + ], + "multifield/wyrd:place": [ + 16, + 21 + ], + "multifield/wyrd:place2": [ + 21, + 21 + ], + "multifield/wyrd:exit": [ + 20, + 21 + ], + "multifield/wyrd:exit2": [ + 20, + 21 + ], + "multifield/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + } + }, + "r3": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0, + "multifield/pooled": [ + 77, + 84 + ], + "multifield/wyrd": [ + 77, + 84 + ], + "multifield/wyrd:place": [ + 16, + 21 + ], + "multifield/wyrd:place2": [ + 21, + 21 + ], + "multifield/wyrd:exit": [ + 20, + 21 + ], + "multifield/wyrd:exit2": [ + 20, + 21 + ], + "multifield/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + } + }, + "r4": { + "scores": { + "single/pooled": [ + 240, + 259 + ], + "single/authored144": [ + 132, + 144 + ], + "single/perturbations108": [ + 100, + 108 + ], + "single/cicada-w1": [ + 29, + 31 + ], + "single/cicada-w2": [ + 22, + 31 + ], + "single/wyrd": [ + 79, + 84 + ], + "single/wyrd:place": [ + 19, + 21 + ], + "single/wyrd:place2": [ + 21, + 21 + ], + "single/wyrd:exit": [ + 19, + 21 + ], + "single/wyrd:exit2": [ + 20, + 21 + ], + "single/failures": 0, + "rotations/pooled": [ + 236, + 259 + ], + "rotations/authored144": [ + 130, + 144 + ], + "rotations/perturbations108": [ + 99, + 108 + ], + "rotations/cicada-w1": [ + 29, + 31 + ], + "rotations/cicada-w2": [ + 22, + 31 + ], + "rotations/wyrd": [ + 77, + 84 + ], + "rotations/wyrd:place": [ + 18, + 21 + ], + "rotations/wyrd:place2": [ + 21, + 21 + ], + "rotations/wyrd:exit": [ + 19, + 21 + ], + "rotations/wyrd:exit2": [ + 19, + 21 + ], + "rotations/failures": 0, + "multifield/pooled": [ + 77, + 84 + ], + "multifield/wyrd": [ + 77, + 84 + ], + "multifield/wyrd:place": [ + 16, + 21 + ], + "multifield/wyrd:place2": [ + 21, + 21 + ], + "multifield/wyrd:exit": [ + 20, + 21 + ], + "multifield/wyrd:exit2": [ + 20, + 21 + ], + "multifield/failures": 0 + }, + "negative": { + "n": 144, + "same_top_as_unrotated": 10, + "follows_description": 122, + "vs_original_gold": 14 + } + } + }, + "vs_bench_row_by_row": { + "r1": { + "single": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "negative": { + "bench_repeat": "r1", + "rows": 144, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "multifield": { + "bench_repeat": "r2", + "rows": 84, + "top_differs": 1, + "flipped_ids": [ + "wyrd-scene-fletcher/f-leave-town/place" + ], + "max_dp": 0.2942167277107136 + } + }, + "r2": { + "single": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "negative": { + "bench_repeat": "r1", + "rows": 144, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "multifield": { + "bench_repeat": "r2", + "rows": 84, + "top_differs": 1, + "flipped_ids": [ + "wyrd-scene-fletcher/f-leave-town/place" + ], + "max_dp": 0.2942167277107136 + } + }, + "r3": { + "single": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "negative": { + "bench_repeat": "r1", + "rows": 144, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "multifield": { + "bench_repeat": "r2", + "rows": 84, + "top_differs": 1, + "flipped_ids": [ + "wyrd-scene-fletcher/f-leave-town/place" + ], + "max_dp": 0.2942167277107136 + } + }, + "oom": { + "single": { + "bench_repeat": "r1", + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "bench_repeat": "r1", + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + }, + "negative": { + "bench_repeat": "r1", + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + }, + "multifield": { + "bench_repeat": "r2", + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + } + } + }, + "across_restarts": { + "oom~r1": { + "single": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + } + }, + "oom~r2": { + "single": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + } + }, + "oom~r3": { + "single": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "rows": 0, + "top_differs": 0, + "flipped_ids": [], + "max_dp": null + } + }, + "r1~r2": { + "single": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + } + }, + "r1~r3": { + "single": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + } + }, + "r2~r3": { + "single": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + }, + "rotations": { + "rows": 560, + "top_differs": 0, + "flipped_ids": [], + "max_dp": 0.0 + } + } + }, + "summary": { + "multifield/pooled": { + "ours_median": 78, + "ours_all": [ + 78, + 78, + 78 + ], + "bench_all": [ + 77, + 77, + 77 + ], + "n": 84 + }, + "multifield/wyrd": { + "ours_median": 78, + "ours_all": [ + 78, + 78, + 78 + ], + "bench_all": [ + 77, + 77, + 77 + ], + "n": 84 + }, + "multifield/wyrd:exit": { + "ours_median": 20, + "ours_all": [ + 20, + 20, + 20 + ], + "bench_all": [ + 20, + 20, + 20 + ], + "n": 21 + }, + "multifield/wyrd:exit2": { + "ours_median": 20, + "ours_all": [ + 20, + 20, + 20 + ], + "bench_all": [ + 20, + 20, + 20 + ], + "n": 21 + }, + "multifield/wyrd:place": { + "ours_median": 17, + "ours_all": [ + 17, + 17, + 17 + ], + "bench_all": [ + 16, + 16, + 16 + ], + "n": 21 + }, + "multifield/wyrd:place2": { + "ours_median": 21, + "ours_all": [ + 21, + 21, + 21 + ], + "bench_all": [ + 21, + 21, + 21 + ], + "n": 21 + }, + "rotations/authored144": { + "ours_median": 130, + "ours_all": [ + 130, + 130, + 130 + ], + "bench_all": [ + 130, + 130, + 130, + 130 + ], + "n": 144 + }, + "rotations/cicada-w1": { + "ours_median": 29, + "ours_all": [ + 29, + 29, + 29 + ], + "bench_all": [ + 29, + 29, + 29, + 29 + ], + "n": 31 + }, + "rotations/cicada-w2": { + "ours_median": 22, + "ours_all": [ + 22, + 22, + 22 + ], + "bench_all": [ + 22, + 22, + 22, + 22 + ], + "n": 31 + }, + "rotations/perturbations108": { + "ours_median": 99, + "ours_all": [ + 99, + 99, + 99 + ], + "bench_all": [ + 99, + 99, + 99, + 99 + ], + "n": 108 + }, + "rotations/pooled": { + "ours_median": 236, + "ours_all": [ + 236, + 236, + 236 + ], + "bench_all": [ + 236, + 236, + 236, + 236 + ], + "n": 259 + }, + "rotations/wyrd": { + "ours_median": 77, + "ours_all": [ + 77, + 77, + 77 + ], + "bench_all": [ + 77, + 77, + 77, + 77 + ], + "n": 84 + }, + "rotations/wyrd:exit": { + "ours_median": 19, + "ours_all": [ + 19, + 19, + 19 + ], + "bench_all": [ + 19, + 19, + 19, + 19 + ], + "n": 21 + }, + "rotations/wyrd:exit2": { + "ours_median": 19, + "ours_all": [ + 19, + 19, + 19 + ], + "bench_all": [ + 19, + 19, + 19, + 19 + ], + "n": 21 + }, + "rotations/wyrd:place": { + "ours_median": 18, + "ours_all": [ + 18, + 18, + 18 + ], + "bench_all": [ + 18, + 18, + 18, + 18 + ], + "n": 21 + }, + "rotations/wyrd:place2": { + "ours_median": 21, + "ours_all": [ + 21, + 21, + 21 + ], + "bench_all": [ + 21, + 21, + 21, + 21 + ], + "n": 21 + }, + "single/authored144": { + "ours_median": 132.0, + "ours_all": [ + 132, + 132, + 132, + 132 + ], + "bench_all": [ + 132, + 132, + 132, + 132 + ], + "n": 144 + }, + "single/cicada-w1": { + "ours_median": 29.0, + "ours_all": [ + 29, + 29, + 29, + 29 + ], + "bench_all": [ + 29, + 29, + 29, + 29 + ], + "n": 31 + }, + "single/cicada-w2": { + "ours_median": 22.0, + "ours_all": [ + 22, + 22, + 22, + 22 + ], + "bench_all": [ + 22, + 22, + 22, + 22 + ], + "n": 31 + }, + "single/perturbations108": { + "ours_median": 100.0, + "ours_all": [ + 100, + 100, + 100, + 100 + ], + "bench_all": [ + 100, + 100, + 100, + 100 + ], + "n": 108 + }, + "single/pooled": { + "ours_median": 240.0, + "ours_all": [ + 240, + 240, + 240, + 240 + ], + "bench_all": [ + 240, + 240, + 240, + 240 + ], + "n": 259 + }, + "single/wyrd": { + "ours_median": 79.0, + "ours_all": [ + 79, + 79, + 79, + 79 + ], + "bench_all": [ + 79, + 79, + 79, + 79 + ], + "n": 84 + }, + "single/wyrd:exit": { + "ours_median": 19.0, + "ours_all": [ + 19, + 19, + 19, + 19 + ], + "bench_all": [ + 19, + 19, + 19, + 19 + ], + "n": 21 + }, + "single/wyrd:exit2": { + "ours_median": 20.0, + "ours_all": [ + 20, + 20, + 20, + 20 + ], + "bench_all": [ + 20, + 20, + 20, + 20 + ], + "n": 21 + }, + "single/wyrd:place": { + "ours_median": 19.0, + "ours_all": [ + 19, + 19, + 19, + 19 + ], + "bench_all": [ + 19, + 19, + 19, + 19 + ], + "n": 21 + }, + "single/wyrd:place2": { + "ours_median": 21.0, + "ours_all": [ + 21, + 21, + 21, + 21 + ], + "bench_all": [ + 21, + 21, + 21, + 21 + ], + "n": 21 + } + } +} diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/checks.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/checks.json new file mode 100644 index 0000000..60f91f4 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/checks.json @@ -0,0 +1,72 @@ +{ + "url": "http://127.0.0.1:18033", + "started_utc": "2026-09-30T16:34:04Z", + "health": { + "status": "ok", + "model": { + "name": "Intern-Decision-4B", + "source": "internlm/Intern-Decision-4B", + "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "checkpoint": "/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "inference_py_sha256": "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863", + "temperature": 1.99241824, + "dtype": "bfloat16", + "attn_implementation": "sdpa", + "device": "cuda", + "max_length": 8192, + "torch_version": "2.10.0+cu128", + "transformers_version": "5.17.0", + "vision_tower": "removed", + "device_name": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", + "allocated_gib": 7.937, + "reserved_gib": 7.969, + "max_reserved_gib": 8.861 + }, + "vram_cap_gib": 8.9, + "max_tokens": 8192, + "max_decisions": 64, + "max_questions_per_call": 16, + "chunking": "/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.", + "workloads": [] + }, + "maxreq": { + "state_chars": 9508, + "tokens_per_call": null, + "decisions": 64, + "options_each": 16, + "body_bytes": 101019, + "runs": [ + { + "status": 503, + "e2e_ms": 193.4, + "code": "out_of_memory", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 7.969, + "max_reserved_gib": 8.861, + "t_end": 1790786053.86457 + } + ], + "t_start": 1790786044.040999, + "t_end": 1790786053.8645778 + }, + "oom": { + "oom_status": 503, + "oom_error": { + "code": "out_of_memory", + "message": "CUDA out of memory. Tried to allocate 64.00 MiB. GPU 0 has a total capacity of 94.97 GiB of which 85.46 GiB is free. Including non-PyTorch memory, this process has 9.50 GiB memory in use. 8.90 GiB allowed; Of the allocated memory 8.80 GiB is allocated by PyTorch, and 57.34 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)" + }, + "oom_ms": 185.5, + "reserved_gib": { + "before": 7.969, + "after_oom": 7.969, + "after_next_request": 7.975 + }, + "next_request_status": 200, + "next_top": "yes", + "pass": true, + "t_start": 1790786053.864641, + "t_end": 1790786054.1036348 + }, + "finished_utc": "2026-09-30T16:34:14Z" +} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/sets.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/sets.json new file mode 100644 index 0000000..afaaa7d --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/sets.json @@ -0,0 +1 @@ +{"label": "oomcap", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:34:14Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.23722597211599, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 37.54}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41475101560354, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.4}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60182298719883, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.98}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.148128962144256, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.33}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.222439888864756, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.57}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39509196020663, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.7}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52464312128723, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.76}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09907806478441, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.34}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.381431087851524, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.51}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8156349286437, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.09}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.320130970329046, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.58}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.948106018826365, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 34.27}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38309186883271, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.64}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81180509738624, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.12}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30958001688123, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 34.6}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9753569830209, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.26}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30314993299544, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.56}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03837702050805, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.24}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42932202294469, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.62}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09882707335055, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.3}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38433099351823, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.71}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02024603262544, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.16}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56383387371898, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.64}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219228921458125, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.51}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47240197844803, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.75}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.92493499815464, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 39.96}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.093718932941556, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 35.28}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98281797207892, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 35.3}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.509343983605504, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 35.84}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.301220862194896, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.58}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28869201242924, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 35.49}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39951287955046, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 35.58}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6781960260123, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 35.93}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.41461311094463, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 35.64}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32499100826681, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 35.62}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.973105976358056, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.27}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62668394483626, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.75}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89747606404126, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.11}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41311212070286, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.62}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40755212493241, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.46}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.56027405709028, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 35.78}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.374661071226, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.48}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13059797883034, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.49}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.281839875504375, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.45}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92189690284431, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 35.03}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85469483025372, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.08}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.292209919542074, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.38}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13972999341786, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 35.26}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.809207966551185, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 35.95}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.16693997941911, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 35.48}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.27066090889275, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 35.61}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.10907993465662, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 35.36}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.68022607639432, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 35.79}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39140199124813, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 35.64}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.41776307672262, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 35.71}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.45210303552449, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 35.74}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.75626716762781, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 35.93}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.37587311863899, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 35.62}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.55350497178733, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 35.88}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.278331980109215, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 35.56}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.541654823347926, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 35.77}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28664100728929, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 35.49}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38413301855326, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 35.62}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.17991097457707, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 35.47}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51268393918872, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 35.63}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.36064296774566, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 35.67}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.62329595722258, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 35.93}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.19341096095741, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 35.5}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59798493795097, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 35.92}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.12647983431816, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 35.38}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.12159102037549, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 36.13}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.213969906792045, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 35.37}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.64391497336328, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 35.8}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15719997510314, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 35.35}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.37492293678224, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 35.47}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5087039321661, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 35.49}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.87483913078904, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 36.88}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.701486075297, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 35.7}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.76746702007949, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 35.9}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.46478299237788, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 35.48}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.500963943079114, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 35.58}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.355602882802486, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 35.46}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.43852295540273, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 35.48}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.54936407878995, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 35.58}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.746176986023784, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 35.88}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.609656039625406, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 35.76}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7645079754293, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 35.9}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.484642980620265, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 35.52}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7450170237571, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 36.01}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.58252498321235, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 35.7}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.53966388851404, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 35.65}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.532903883606195, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 35.72}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.45397406257689, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 35.55}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.18007405102253, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 37.14}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.83250686340034, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 35.77}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.141730008646846, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 35.31}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.711885921657085, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 35.86}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.2578510325402, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 35.39}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.44646317884326, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 35.6}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.643134128302336, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.72}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.369539961218834, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.54}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08837786689401, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.19}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33514994382858, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.49}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.929765017703176, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.11}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53050300292671, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.63}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34834096394479, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.53}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19118796288967, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.32}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.505812011659145, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.59}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1606289409101, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.36}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37414092756808, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.43}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.387041142210364, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.52}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.097027990967035, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.22}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34170086681843, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.46}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01535705476999, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.22}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63216291554272, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.81}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2465499676764, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.33}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30636090040207, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.45}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14422881603241, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.19}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42505190894008, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.47}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90437483601272, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.11}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.274840055033565, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.5}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0169070083648, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.22}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24615997634828, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.42}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31174999848008, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.47}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27444005198777, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.37}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96497599594295, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.15}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3413001652807, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.43}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17161901295185, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.2}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.056667963042855, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.25}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00762614607811, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.16}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.289900006726384, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.41}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48924215324223, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.78}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.277380004525185, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.33}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32143994420767, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.45}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3251608107239, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.5}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.320510018616915, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.53}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.399220978841186, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.77}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18486800789833, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.44}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66330401413143, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.84}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19314909540117, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.5}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24647895433009, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.5}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94557584822178, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.13}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.162937922403216, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.51}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01731702312827, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.33}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.623084150254726, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.96}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96165606379509, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 34.29}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29939996264875, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.52}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.271609995514154, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.57}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9182659778744, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.24}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.558943033218384, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 34.77}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53821295686066, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 34.7}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12768782861531, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.26}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52825199626386, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 34.72}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.092468002811074, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.41}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.181489169597626, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.55}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44786186143756, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 34.73}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35298989340663, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.69}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06193692050874, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.38}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.01111094839871, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 35.85}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.68903615511954, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.84}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.642693845555186, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 34.81}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49979310669005, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 34.72}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31518005952239, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 34.44}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.553382804617286, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 34.71}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35398990102112, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 34.61}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.049086997285485, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.32}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39555086940527, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 34.61}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.181448889896274, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.48}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27221884764731, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.47}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366241216659546, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 34.56}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.330060033127666, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 34.56}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.488111993297935, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.56}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.277020048350096, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 34.54}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45296294800937, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 34.6}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.453820928931236, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 34.63}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37088106572628, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.53}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34380006790161, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 34.57}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54039295017719, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 34.8}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20557899214327, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 34.46}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5595330260694, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 34.72}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.394262151792645, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 34.69}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18284796923399, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 34.48}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.785826018080115, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 34.98}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15310897678137, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.43}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28171996772289, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 34.48}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12688805535436, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 34.38}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66276501119137, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.82}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53853286616504, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 34.71}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36784099601209, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 34.54}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.466422887519, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.59}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27469011209905, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 34.61}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21635905094445, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.48}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44870191439986, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 34.67}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29078103788197, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 34.63}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65176306478679, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 34.66}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.295239977538586, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.53}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76869596913457, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 34.87}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63848403468728, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 34.69}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67106402665377, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 34.82}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.814775947481394, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 34.78}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76543587446213, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.89}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58483310230076, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.61}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71311487071216, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 34.86}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.40134691260755, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 38.53}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2775299474597, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 34.37}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32293006032705, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.45}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87099686451256, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 34.88}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.351120030507445, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 34.64}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62671397998929, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 34.87}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.993148900568485, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 35.17}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.722705164924264, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.76}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.437052000314, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 34.56}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71920409053564, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 34.89}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22933982498944, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 34.36}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.332889856770635, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 34.53}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.308149971067905, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 34.47}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.554593058303, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.89}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38051187247038, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 34.69}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4844828620553, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 34.78}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.238848861306906, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.56}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.328319856897, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 34.6}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32498991303146, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 34.66}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.504232954233885, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.8}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30669105425477, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.5}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.938406923785806, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 35.25}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63057407923043, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.74}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.274949019774795, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 34.63}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65260404907167, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 34.94}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17986890114844, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 34.49}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01054700650275, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.34}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.894396809861064, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 35.11}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20629890449345, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.54}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25675889290869, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 34.56}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48929291032255, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 34.69}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39581210538745, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 34.75}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.196108140051365, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 34.36}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64725383371115, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 34.86}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.210228852927685, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 34.51}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68860409036279, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 34.87}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47408208437264, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 34.75}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56812307983637, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 34.78}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.394850980490446, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 34.52}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.202089093625546, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 34.45}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38056984730065, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.6}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47487301006913, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 34.63}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.200018065050244, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.34}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63307398185134, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.7}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.132757948711514, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 34.31}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.868306040763855, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 34.87}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06219792366028, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.31}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19212906435132, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 34.31}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53431304357946, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.55}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.19284099154174, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 35.34}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07225690595806, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.16}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.920276867225766, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 35.09}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26090909913182, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.25872088782489, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 35.28}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17040806077421, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.721164990216494, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 34.78}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.370400968939066, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.5}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.944956915453076, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 35.0}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28626994416118, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.45}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78369491919875, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98966698721051, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.22}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59627300314605, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.77}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23316895775497, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93475707061589, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 35.09}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3845009598881, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.913707084953785, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 35.11}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29499108158052, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.29}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84358687698841, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 35.05}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1690490283072, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.25}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.9963091108948, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 35.03}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3159209266305, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.44}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46665199287236, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 34.52}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00981708243489, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.17}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85625612363219, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 34.98}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29233983904123, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71119401790202, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 34.84}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02673711813986, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.781886057928205, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 34.95}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18265891075134, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.37}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23824995197356, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 35.24}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.416460923850536, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.56}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.9880191758275, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 35.19}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.295280024409294, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5001040995121, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 35.54}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47580200433731, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.57}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.17253094166517, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 35.16}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50711283646524, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.66}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.05902902968228, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 35.2}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39314083941281, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.52}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.18237013183534, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 35.21}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3239500913769, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.967418087646365, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 35.19}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.393131060525775, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.879507057368755, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 35.02}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69618389196694, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.9}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.21244090795517, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 35.41}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19574888236821, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.56}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90636686608195, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 35.22}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37120097503066, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.96049696393311, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 35.26}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.363520823419094, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6785141415894, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 35.01}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54643304273486, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.69}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.20465109311044, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 35.42}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.157318787649274, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93897689320147, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 35.29}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28582989424467, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.044807966798544, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 35.24}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14055777527392, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.32}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.012698059901595, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 35.04}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.578893126919866, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.76}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98895818181336, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 35.2}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34340099617839, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.54}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.247771095484495, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 35.34}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.392440950497985, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.22628105990589, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 35.32}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27182000875473, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.5}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76667593047023, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 34.91}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.241159938275814, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66026510857046, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.632894003763795, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76150499470532, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.167889066040516, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.5}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.507982924580574, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 34.85}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43492090143263, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.65}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.882277043536305, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41633193567395, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.75}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77905507013202, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 34.98}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31067990884185, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.5}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70685498602688, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 34.84}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.138188956305385, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.49}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68459497950971, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.197329103946686, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51575294882059, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28022985160351, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.61}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41095098666847, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 34.73}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06585709005594, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.39}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85259695723653, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35737991519272, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81310492195189, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 34.86}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3339899815619, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.62}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62741503305733, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34614094719291, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.48}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39145118743181, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 34.63}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.293750094249845, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.36}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58715302683413, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 34.83}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22489904612303, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.5}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56980295293033, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03207685612142, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.22}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54350310191512, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 34.74}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.324680015444756, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.52}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.812156137079, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 34.89}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.6053320299834, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 40.23}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.28148593008518, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 37.9}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.29975279606879, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 36.6}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.51967600733042, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 36.69}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.720316952094436, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.94}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69688604027033, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 35.98}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59358311071992, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.814185954630375, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4731719493866, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63327398337424, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.400661036372185, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.72}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56035305559635, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 34.85}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45980188250542, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.65}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63622301630676, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59067296050489, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.73}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66763396561146, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 34.8}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54875296726823, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60449308902025, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184438202530146, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.763225845992565, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26228000596166, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.750134943053126, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 34.94}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28356002829969, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.64}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79680607840419, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38389117643237, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.6}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.631102837622166, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 34.81}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.536223186179996, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.62}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62908396311104, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 34.9}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.275920156389475, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.52}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58123307488859, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27397997863591, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.59}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69487398676574, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.185578839853406, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.51}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45308192446828, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 34.66}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00314718298614, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.35}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28927904739976, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 34.6}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96490591391921, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.3}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.2695679590106, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 39.43}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.326201098039746, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.49}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.97449001669884, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 36.15}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.183409091085196, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.38}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.18282997235656, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 35.37}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97930695489049, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.17}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6505660135299, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 35.81}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62237308546901, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.82}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.48197404295206, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 35.69}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.457781843841076, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.79}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.84954797104001, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 35.98}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.442742148414254, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.59}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69267599657178, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 35.78}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.218199111521244, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.46}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.70195709168911, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 35.84}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60918392799795, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.78942714072764, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.648963974788785, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.86}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.652455899864435, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63548377901316, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.84}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.62937516532838, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 35.7}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19958895817399, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.48}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.90152811072767, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 36.07}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65335492789745, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.704317063093185, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 35.97}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5112930778414, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.78}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.81019679829478, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 36.1}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8879071213305, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.07}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.77075691707432, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 35.99}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34439006075263, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.69}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.20458210632205, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 36.32}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90418687276542, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.94073888659477, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 36.15}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47088206000626, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.65}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.017789997160435, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 36.2}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84730611369014, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.03}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65421493537724, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 35.82}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.079717963933945, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.36}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.807767909020185, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 36.08}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.630253937095404, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69877592660487, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 35.9}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35887003131211, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.68}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.714067079126835, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42013196460903, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.73}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.33454195410013, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 35.64}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15865802951157, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.46}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.05833000317216, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 36.18}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.80384594574571, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.89}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.704387145116925, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 35.87}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.390201119706035, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.67}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51665393449366, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 35.76}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47215215116739, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.8}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23736100457609, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 35.46}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05058712512255, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.4}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6818558908999, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 35.98}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41750190965831, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.75}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.481794064864516, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.73}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30482002720237, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.66}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.929578848183155, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 36.12}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62770397588611, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.3920029271394, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 35.63}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14828798547387, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.5}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.819047855213284, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 36.05}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75407513603568, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.02}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.52751501835883, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 35.87}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.509972928091884, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.78}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.488622987642884, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 35.78}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45463187806308, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.72}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15074893645942, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13249904103577, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.42}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.49466401152313, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 35.79}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27120906114578, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.56}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.87096792459488, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 36.1}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20880905166268, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.51}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.458554074168205, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 35.74}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28797985240817, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.56}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.33847213536501, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16833796352148, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.4}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.62128616310656, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 35.89}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37602103501558, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.626525085419416, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 35.88}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31860010884702, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.64200599491596, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 35.84}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36227983608842, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.58}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.20172999985516, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 35.49}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61504404246807, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.97}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35280193015933, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 35.6}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10500802844763, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.41}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.60924509353936, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 35.87}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.190100161358714, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 36.91}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.37616206146777, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.414581049233675, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.65}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15926006063819, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43279096670449, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.7}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.101378828287125, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 35.35}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26288001053035, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.49}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.607246009632945, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.9}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32368000783026, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.63}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.42518292367458, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17890894040465, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.45}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.265581941232085, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.56}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32330002821982, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.62}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6166051030159, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 35.75}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21982892602682, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.55}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.41102309525013, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 35.73}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33470989391208, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.84435794316232, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 36.0}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219559075310826, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.44}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32687181420624, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 35.5}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29221993125975, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.63}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.43747288919985, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 35.64}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.451210897415876, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.77}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7965370900929, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 35.95}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34256117604673, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.71949599124491, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 35.88}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.457001861184835, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.60106482170522, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.8}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.585144164040685, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.67}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.653355091810226, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.86}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50413190573454, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.54144411161542, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 35.81}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45772200636566, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.54635497555137, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 35.82}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19893889501691, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.56}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35792210884392, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 35.59}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.427420960739255, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.67}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.68396603502333, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 35.95}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45194198563695, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65468502044678, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 35.83}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.262788973748684, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.73790707439184, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.87}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.308130180463195, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.62}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.29981107264757, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 35.61}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40749089047313, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.69}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.701536133885384, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 35.87}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67247498035431, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.83}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.2810418959707, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 35.62}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.445461843162775, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.74}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.528054021298885, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 35.7}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.145528011024, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.47}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.326611975207925, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 35.59}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29776991531253, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51731400936842, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 35.79}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366000935435295, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.31434203311801, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 35.6}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.925781078636646, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.26}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.838288048282266, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 35.96}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.506932159885764, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.69}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.561325054615736, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 35.69}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.583003053441644, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.86}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35709183476865, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 35.66}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36696997657418, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.64}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32290298305452, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 35.62}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.378090899437666, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.62}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.91890915110707, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 36.16}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.506822030991316, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.77}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.395293056964874, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 35.71}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39539091475308, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.57}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.40112290345132, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 35.64}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61243401095271, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.71}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65101491101086, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 35.73}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34878217801452, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.430113112553954, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 35.69}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41500107571483, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5201838798821, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 35.7}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.296750117093325, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.61}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.323771907016635, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.56}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40347200073302, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.75}], "seconds": 21.3, "failures": 0}}, "finished_utc": "2026-09-30T16:34:35Z"} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/vram.csv new file mode 100644 index 0000000..98804e3 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/oom/vram.csv @@ -0,0 +1,305 @@ +# pid=1895116 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790786044.168189323,8812,8822 +1790786044.269045856,8812,8822 +1790786044.372432028,8812,8822 +1790786044.473463003,8812,8822 +1790786044.576815665,8812,8822 +1790786044.679750302,8812,9408 +1790786044.785333909,9398,9664 +1790786044.896467090,9654,9664 +1790786044.998121902,9654,9664 +1790786045.112920366,9654,9664 +1790786045.212860788,9654,9664 +1790786045.320714162,9654,9664 +1790786045.431997525,9654,9664 +1790786045.538023048,9654,9664 +1790786045.641723804,9654,9664 +1790786045.748181851,9654,9664 +1790786045.854809780,9654,9664 +1790786045.961133296,9654,9664 +1790786046.061937629,9654,9664 +1790786046.165827987,9654,9664 +1790786046.268673833,9654,9664 +1790786046.371860622,9654,9664 +1790786046.473481164,9654,9664 +1790786046.580217965,9654,9664 +1790786046.689452924,9654,9664 +1790786046.805418922,9654,9664 +1790786046.905598447,9654,9664 +1790786047.017627849,9654,9664 +1790786047.120874469,9654,9664 +1790786047.230576744,9654,9664 +1790786047.335459223,9654,9664 +1790786047.438058406,9654,9664 +1790786047.549733544,9654,9664 +1790786047.653074466,9654,9664 +1790786047.757390729,9654,9664 +1790786047.858761167,9654,9664 +1790786047.969951860,9654,9664 +1790786048.071047405,9654,9664 +1790786048.182224527,9654,9664 +1790786048.287195878,9654,9664 +1790786048.395618367,9654,9664 +1790786048.499898120,9654,9664 +1790786048.610741888,9654,9664 +1790786048.711710232,9654,9664 +1790786048.825872839,9654,9664 +1790786048.926662871,9654,9664 +1790786049.037905614,9654,9664 +1790786049.147696630,9654,9664 +1790786049.264543407,9654,9664 +1790786049.364955905,9654,9664 +1790786049.478969159,9654,9664 +1790786049.584457676,9654,9664 +1790786049.694840648,9654,9664 +1790786049.801375167,9654,9664 +1790786049.911595068,9654,9664 +1790786050.016114563,9654,9664 +1790786050.130576363,9654,9664 +1790786050.234037256,9654,9664 +1790786050.344658831,9654,9664 +1790786050.450143318,9654,9664 +1790786050.551524216,9654,9664 +1790786050.664520669,9654,9664 +1790786050.765953109,9654,9664 +1790786050.878236234,9654,9664 +1790786050.981459074,9654,9664 +1790786051.096501661,9654,9664 +1790786051.199314436,9656,9666 +1790786051.300835547,9656,8822 +1790786051.406026129,8812,8822 +1790786051.509378131,8812,8822 +1790786051.609895040,8812,8822 +1790786051.712092209,8812,8822 +1790786051.816476713,8812,8822 +1790786051.917808011,9724,9734 +1790786052.019315421,9724,8822 +1790786052.119740179,9724,9734 +1790786052.223558646,9724,8822 +1790786052.324514570,8812,8822 +1790786052.422031205,8812,8822 +1790786052.525959133,8812,8822 +1790786052.630112744,9466,9734 +1790786052.733610577,9724,9734 +1790786052.836694296,8812,9734 +1790786052.939251959,9724,9734 +1790786053.041757901,8812,8822 +1790786053.145840141,8812,8822 +1790786053.248643356,8812,8822 +1790786053.352541774,8812,9734 +1790786053.455574442,9724,9734 +1790786053.559617072,8812,8822 +1790786053.660222462,8812,8822 +1790786053.761831704,9724,9734 +1790786053.863925281,9724,9086 +1790786053.965529572,9724,9734 +1790786054.071097600,9724,8822 +1790786054.171271045,8818,8828 +1790786054.275709689,8818,8828 +1790786054.380262634,8818,8828 +1790786054.483105170,8818,8828 +1790786054.582680018,8818,8828 +1790786054.687411036,8818,8828 +1790786054.786901303,8818,8828 +1790786054.884256586,8818,8828 +1790786054.985041088,8818,8828 +1790786055.087011494,8818,8828 +1790786055.193773565,8818,8828 +1790786055.299059149,8818,8828 +1790786055.397299392,8818,8828 +1790786055.503216793,8818,8828 +1790786055.606199880,8818,8828 +1790786055.715536871,8818,8828 +1790786055.820981967,8818,8828 +1790786055.923801362,8818,8828 +1790786056.026964242,8818,8828 +1790786056.128920418,8818,8828 +1790786056.231365749,8818,8828 +1790786056.332869409,8818,8828 +1790786056.437235122,8818,8828 +1790786056.541403524,8818,8828 +1790786056.642243776,8818,8828 +1790786056.747418689,8818,8828 +1790786056.849783970,8818,8828 +1790786056.953331143,8818,8828 +1790786057.058014191,8818,8828 +1790786057.165179146,8834,8844 +1790786057.268708500,8834,8844 +1790786057.373516379,8834,8844 +1790786057.476896190,8834,8844 +1790786057.578053537,8834,8844 +1790786057.682023276,8834,8844 +1790786057.783938721,8834,8844 +1790786057.890467309,8834,8844 +1790786057.990107818,8834,8844 +1790786058.093797764,8834,8844 +1790786058.199919207,8834,8844 +1790786058.302911355,8834,8844 +1790786058.403915150,8834,8844 +1790786058.508407795,8834,8844 +1790786058.613612297,8834,8844 +1790786058.716186740,8834,8844 +1790786058.820472583,8834,8844 +1790786058.924975597,8834,8844 +1790786059.027746783,8834,8844 +1790786059.132616542,8834,8844 +1790786059.235259836,8834,8844 +1790786059.339490907,8834,8844 +1790786059.441716116,8834,8844 +1790786059.547787949,8834,8844 +1790786059.646887872,8834,8844 +1790786059.748900978,8834,8844 +1790786059.852270180,8834,8844 +1790786059.948982975,8834,8844 +1790786060.051093842,8834,8844 +1790786060.154347103,8834,8844 +1790786060.263682444,8834,8844 +1790786060.368226619,8834,8844 +1790786060.469250154,8834,8844 +1790786060.573891811,8834,8844 +1790786060.680670902,8834,8844 +1790786060.786210109,8834,8844 +1790786060.889288377,8834,8844 +1790786060.993480109,8834,8844 +1790786061.093702394,8834,8844 +1790786061.197108087,8834,8844 +1790786061.301101186,8834,8844 +1790786061.405135455,8834,8844 +1790786061.509453878,8834,8844 +1790786061.610066308,8834,8844 +1790786061.716237183,8834,8844 +1790786061.821237123,8834,8844 +1790786061.927098934,8834,8846 +1790786062.029145241,8836,8846 +1790786062.133937769,8836,8846 +1790786062.235259337,8836,8846 +1790786062.336842508,8836,8846 +1790786062.440487194,8836,8846 +1790786062.543613103,8836,8846 +1790786062.648410792,8836,8846 +1790786062.750901463,8836,8846 +1790786062.852518114,8836,8846 +1790786062.950795868,8836,8846 +1790786063.051403048,8836,8846 +1790786063.157206538,8836,8846 +1790786063.259705940,8836,8846 +1790786063.365974385,8836,8846 +1790786063.471252549,8836,8846 +1790786063.577268142,8836,8846 +1790786063.678367507,8836,8846 +1790786063.781554837,8836,8846 +1790786063.885537526,8836,8846 +1790786063.989090030,8836,8846 +1790786064.093393853,8836,8846 +1790786064.196789855,8836,8846 +1790786064.301286970,8836,8846 +1790786064.407359153,8836,8846 +1790786064.511490634,8836,8846 +1790786064.613120706,8836,8846 +1790786064.716076963,8836,8846 +1790786064.817650584,8836,8846 +1790786064.920314248,8836,8846 +1790786065.021658356,8836,8846 +1790786065.133065881,8836,8846 +1790786065.243060219,8836,8846 +1790786065.349707709,8836,8846 +1790786065.457669054,8836,8846 +1790786065.568920207,8836,8846 +1790786065.679146297,8836,8846 +1790786065.789512790,8836,8846 +1790786065.897483355,8836,8846 +1790786066.005894385,8836,8846 +1790786066.112639866,8836,8846 +1790786066.222999979,8836,8846 +1790786066.326058457,8836,8846 +1790786066.433976721,8836,8846 +1790786066.536550184,8836,8846 +1790786066.641908079,8836,8846 +1790786066.746854599,8836,8846 +1790786066.851266163,8836,8846 +1790786066.951148344,8836,8846 +1790786067.050759323,8836,8846 +1790786067.156182589,8836,8846 +1790786067.261903148,8836,8846 +1790786067.365557163,8836,8846 +1790786067.471905819,8836,8846 +1790786067.575707186,8836,8846 +1790786067.676687901,8836,8846 +1790786067.783223339,8836,8846 +1790786067.884850041,8836,8846 +1790786067.988971061,8836,8846 +1790786068.088866613,8836,8846 +1790786068.193651631,8836,8846 +1790786068.299613613,8836,8846 +1790786068.402757972,8836,8846 +1790786068.507275538,8836,8846 +1790786068.613054377,8836,8846 +1790786068.712147500,8836,8846 +1790786068.814360299,8836,8846 +1790786068.915408794,8836,8846 +1790786069.020393174,8836,8846 +1790786069.127535500,8856,8866 +1790786069.233427461,8856,8866 +1790786069.335585219,8856,8866 +1790786069.443437462,8856,8866 +1790786069.548328132,8856,8866 +1790786069.650126556,8856,8866 +1790786069.753833921,8856,8866 +1790786069.853937405,8856,8866 +1790786069.957672372,8856,8866 +1790786070.057627284,8856,8866 +1790786070.161767395,8856,8866 +1790786070.261364954,8856,8866 +1790786070.363176357,8856,8866 +1790786070.468890456,8856,8866 +1790786070.573291530,8856,8866 +1790786070.677258979,8856,8866 +1790786070.781375149,8856,8866 +1790786070.888336513,8856,8866 +1790786070.992676276,8856,8866 +1790786071.097718157,8856,8866 +1790786071.200188529,8856,8866 +1790786071.303544610,8856,8866 +1790786071.408397909,8856,8866 +1790786071.513294989,8856,8866 +1790786071.613724697,8856,8866 +1790786071.719297184,8856,8866 +1790786071.815557484,8856,8866 +1790786071.921545567,8856,8866 +1790786072.020266765,8856,8866 +1790786072.121251479,8856,8866 +1790786072.223477288,8856,8866 +1790786072.328918803,8856,8866 +1790786072.429870907,8856,8866 +1790786072.531916244,8856,8866 +1790786072.632570735,8856,8866 +1790786072.736258250,8856,8866 +1790786072.836879051,8856,8866 +1790786072.939187560,8856,8866 +1790786073.041071975,8856,8866 +1790786073.143144892,8856,8866 +1790786073.245493662,8856,8866 +1790786073.351254002,8856,8866 +1790786073.457651789,8856,8866 +1790786073.559749976,8856,8866 +1790786073.663030907,8856,8866 +1790786073.764413816,8856,8866 +1790786073.865508312,8856,8866 +1790786073.965860619,8856,8866 +1790786074.064249193,8856,8866 +1790786074.163340846,8856,8866 +1790786074.268479778,8856,8866 +1790786074.374511240,8856,8866 +1790786074.477970804,8856,8866 +1790786074.577967827,8856,8866 +1790786074.681157967,8856,8866 +1790786074.778507819,8856,8866 +1790786074.884194508,8856,8866 +1790786074.987060784,8856,8866 +1790786075.091434837,8856,8866 +1790786075.196457118,8856,8866 +1790786075.299980622,8856,8866 +1790786075.405640210,8856,8866 +1790786075.506410312,8856,8866 +1790786075.614052093,8856,8866 +1790786075.789279953,8856,2 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/checks.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/checks.json new file mode 100644 index 0000000..418afa4 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/checks.json @@ -0,0 +1,82 @@ +{ + "url": "http://127.0.0.1:18033", + "started_utc": "2026-09-30T16:20:35Z", + "health": { + "status": "ok", + "model": { + "name": "Intern-Decision-4B", + "source": "internlm/Intern-Decision-4B", + "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "checkpoint": "/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "inference_py_sha256": "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863", + "temperature": 1.99241824, + "dtype": "bfloat16", + "attn_implementation": "sdpa", + "device": "cuda", + "max_length": 8192, + "torch_version": "2.10.0+cu128", + "transformers_version": "5.17.0", + "vision_tower": "removed", + "device_name": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", + "allocated_gib": 7.937, + "reserved_gib": 7.969, + "max_reserved_gib": 8.861 + }, + "vram_cap_gib": null, + "max_tokens": 8192, + "max_decisions": 64, + "max_questions_per_call": 16, + "chunking": "/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.", + "workloads": [] + }, + "maxreq": { + "state_chars": 11337, + "tokens_per_call": 8192, + "decisions": 64, + "options_each": 16, + "body_bytes": 100953, + "runs": [ + { + "status": 422, + "e2e_ms": 421.3, + "code": "invalid_request", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.531, + "t_end": 1790785244.9259193 + }, + { + "status": 422, + "e2e_ms": 419.9, + "code": "invalid_request", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.531, + "t_end": 1790785245.3499992 + } + ], + "t_start": 1790785235.146321, + "t_end": 1790785245.3500075 + }, + "oom": { + "oom_status": 422, + "oom_error": { + "code": "invalid_request", + "message": "Example has 8202 tokens, above 8192; truncation is forbidden" + }, + "oom_ms": 429.8, + "reserved_gib": { + "before": 7.969, + "after_oom": 7.969, + "after_next_request": 7.975 + }, + "next_request_status": 200, + "next_top": "yes", + "pass": false, + "t_start": 1790785245.3500695, + "t_end": 1790785245.8316922 + }, + "finished_utc": "2026-09-30T16:20:45Z" +} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/maxreq2.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/maxreq2.json new file mode 100644 index 0000000..276d0c7 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/maxreq2.json @@ -0,0 +1,74 @@ +{ + "url": "http://127.0.0.1:18033", + "started_utc": "2026-09-30T16:21:21Z", + "health": { + "status": "ok", + "model": { + "name": "Intern-Decision-4B", + "source": "internlm/Intern-Decision-4B", + "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "checkpoint": "/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "inference_py_sha256": "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863", + "temperature": 1.99241824, + "dtype": "bfloat16", + "attn_implementation": "sdpa", + "device": "cuda", + "max_length": 8192, + "torch_version": "2.10.0+cu128", + "transformers_version": "5.17.0", + "vision_tower": "removed", + "device_name": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", + "allocated_gib": 7.937, + "reserved_gib": 7.975, + "max_reserved_gib": 9.531 + }, + "vram_cap_gib": null, + "max_tokens": 8192, + "max_decisions": 64, + "max_questions_per_call": 16, + "chunking": "/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.", + "workloads": [] + }, + "maxreq": { + "state_chars": 9508, + "tokens_per_call": 8191, + "decisions": 64, + "options_each": 16, + "body_bytes": 101019, + "runs": [ + { + "status": 200, + "e2e_ms": 1641.4, + "code": null, + "calls": 4, + "input_tokens": [ + 8191, + 8191, + 8191, + 8191 + ], + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.531, + "t_end": 1790785293.4182348 + }, + { + "status": 200, + "e2e_ms": 1640.9, + "code": null, + "calls": 4, + "input_tokens": [ + 8191, + 8191, + 8191, + 8191 + ], + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.531, + "t_end": 1790785295.0653996 + } + ], + "t_start": 1790785281.6645088, + "t_end": 1790785295.065496 + }, + "finished_utc": "2026-09-30T16:21:35Z" +} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/overhead.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/overhead.json new file mode 100644 index 0000000..aeaad16 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/overhead.json @@ -0,0 +1,37 @@ +[ + { + "tag": "rest", + "card_mib": 8822, + "reserved_mib": 8160, + "outside_mib": 660, + "max_reserved_mib": 9074 + }, + { + "tag": "after concurrent burst 1", + "card_mib": 8828, + "reserved_mib": 8166, + "outside_mib": 660, + "max_reserved_mib": 9074 + }, + { + "tag": "after concurrent burst 2", + "card_mib": 8828, + "reserved_mib": 8166, + "outside_mib": 660, + "max_reserved_mib": 9074 + }, + { + "tag": "after latency shapes", + "card_mib": 8824, + "reserved_mib": 8160, + "outside_mib": 662, + "max_reserved_mib": 9074 + }, + { + "tag": "after largest requests", + "card_mib": 8830, + "reserved_mib": 8166, + "outside_mib": 662, + "max_reserved_mib": 9760 + } +] \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/shape.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/shape.json new file mode 100644 index 0000000..b835e58 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/shape.json @@ -0,0 +1 @@ +{"label": "probe", "backend": "semif", "runs": 1, "per": 5, "max_questions": 10000, "events": [["start", 1790785217.097767], ["short1:measure", 1790785217.1804233], ["crit21:measure", 1790785217.5361114], ["long1:measure", 1790785224.795359], ["long16:measure", 1790785232.4314742], ["shapes:done", 1790785233.5109885], ["end", 1790785233.5109909]], "shapes": {"short1": {"rows": 1, "runs": [{"e2e_ms": [36.293586948886514, 36.00253281183541, 37.3701888602227, 36.00338310934603, 35.94383201561868], "server_ms": [34.18141300790012, 34.299023915082216, 34.113402012735605, 34.34582380577922, 34.3846851028502], "statuses": [200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.00338310934603, "median_server": 34.299023915082216}], "tokens": 181, "median_e2e_ms": 36.0, "run_medians_e2e_ms": [36.0], "median_server_ms": 34.3, "non_200": 0}, "crit21": {"rows": 21, "runs": [{"e2e_ms": [85.47385083511472, 85.30352916568518, 85.49744100309908, 85.56356094777584, 85.31283889897168], "server_ms": [79.13649803958833, 78.94592499360442, 79.16115806438029, 79.16332804597914, 78.98750598542392], "statuses": [200, 200, 200, 200, 200], "tokens": null, "median_e2e": 85.47385083511472, "median_server": 79.13649803958833}], "tokens": null, "median_e2e_ms": 85.5, "run_medians_e2e_ms": [85.5], "median_server_ms": 79.1, "non_200": 0}, "long1": {"rows": 1, "runs": [{"e2e_ms": [193.0149649269879, 186.22795795090497, 187.29469901882112, 211.8130698800087, 182.15654999949038], "server_ms": [189.49031387455761, 182.79000790789723, 183.78369905985892, 208.4096318576485, 178.73777099885046], "statuses": [200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 187.29469901882112, "median_server": 183.78369905985892}], "tokens": 4019, "median_e2e_ms": 187.3, "run_medians_e2e_ms": [187.3], "median_server_ms": 183.8, "non_200": 0}, "long16": {"rows": 16, "runs": [{"e2e_ms": [212.7648708410561, 229.93756900541484, 210.01248992979527, 212.9045429173857, 211.45822713151574], "server_ms": [207.08801690489054, 224.23489298671484, 204.2714338749647, 207.02347601763904, 205.7694320101291], "statuses": [200, 200, 200, 200, 200], "tokens": null, "median_e2e": 212.7648708410561, "median_server": 207.02347601763904}], "tokens": null, "median_e2e_ms": 212.8, "run_medians_e2e_ms": [212.8], "median_server_ms": 207.0, "non_200": 0}}} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/vram.csv new file mode 100644 index 0000000..0be2c2a --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead-uncapped/vram.csv @@ -0,0 +1,877 @@ +# pid=1849348 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790785210.381560709,8812,8822 +1790785210.490009044,8812,8822 +1790785210.601872697,8812,8822 +1790785210.717730987,8812,8822 +1790785210.824516722,8812,8822 +1790785210.943795281,8812,8822 +1790785211.051771180,8812,8822 +1790785211.162235348,8812,8822 +1790785211.333322301,8812,8822 +1790785211.441870937,8812,8822 +1790785211.547764932,8812,8822 +1790785211.656009584,8812,8822 +1790785211.778490050,8812,8822 +1790785211.886035543,8818,8828 +1790785211.999561516,8818,8828 +1790785212.107175351,8818,8828 +1790785212.214171039,8818,8828 +1790785212.322071797,8818,8828 +1790785212.431437712,8818,8828 +1790785212.543994364,8818,8828 +1790785212.647227119,8818,8828 +1790785212.752386585,8818,8828 +1790785212.860144902,8818,8828 +1790785212.961360753,8818,8828 +1790785213.074807595,8818,8828 +1790785213.175418260,8818,8828 +1790785213.280011960,8818,8828 +1790785213.393008056,8818,8828 +1790785213.497560126,8818,8828 +1790785213.611127569,8818,8828 +1790785213.717094326,8818,8828 +1790785213.824907952,8818,8828 +1790785213.936299191,8818,8828 +1790785214.041692611,8818,8828 +1790785214.152076247,8818,8828 +1790785214.253764144,8818,8828 +1790785214.360277396,8818,8828 +1790785214.478911918,8818,8828 +1790785214.591639591,8818,8828 +1790785214.712161404,8818,8828 +1790785214.816413131,8818,8828 +1790785214.933034709,8818,8828 +1790785215.038134755,8818,8828 +1790785215.141309309,8818,8828 +1790785215.255158215,8818,8828 +1790785215.361958060,8818,8828 +1790785215.480280048,8818,8828 +1790785215.583741645,8818,8828 +1790785215.685142880,8818,8828 +1790785215.790066673,8818,8828 +1790785215.894396340,8818,8828 +1790785216.000368907,8818,8828 +1790785216.105034007,8818,8828 +1790785216.207999079,8818,8828 +1790785216.314139697,8818,8828 +1790785216.417832167,8818,8828 +1790785216.523486949,8818,8828 +1790785216.625370518,8818,8828 +1790785216.725997664,8818,8828 +1790785216.837292941,8818,8828 +1790785216.949085883,8818,8828 +1790785217.069055260,8818,8828 +1790785217.184259502,8818,8828 +1790785217.297021156,8818,8828 +1790785217.408450795,8818,8948 +1790785217.510832210,8938,8948 +1790785217.625230403,8938,8948 +1790785217.729741652,8938,8948 +1790785217.834764397,8938,8948 +1790785217.946009213,8938,8948 +1790785218.049858385,9146,9156 +1790785218.169087623,9402,9412 +1790785218.272008564,9402,9412 +1790785218.377721677,9402,9412 +1790785218.483041636,9402,9412 +1790785218.582735240,9402,9412 +1790785218.724227514,9402,9412 +1790785218.819843831,9402,9412 +1790785219.004492120,9402,9412 +1790785219.107295319,9402,9412 +1790785219.208147177,9402,9412 +1790785219.305508594,9402,9412 +1790785219.400409753,9402,9412 +1790785219.504237634,9402,9412 +1790785219.599332826,9402,9412 +1790785219.697736715,9402,9412 +1790785219.807138850,9402,9412 +1790785219.904320386,9402,9412 +1790785220.015222888,9402,9412 +1790785220.112897519,9402,9412 +1790785220.216813841,9402,9412 +1790785220.328735016,9402,9414 +1790785220.433098893,9404,9414 +1790785220.528839362,9404,9414 +1790785220.636518888,9404,9414 +1790785220.739264127,9404,9414 +1790785220.842384560,9404,9414 +1790785220.947285124,9404,9414 +1790785221.048647667,9404,9414 +1790785221.151142863,9404,9414 +1790785221.248837794,9404,9414 +1790785221.373370153,9404,9414 +1790785221.480254800,9404,9414 +1790785221.592460597,9404,9414 +1790785221.692508065,9404,9414 +1790785221.796296876,9404,9414 +1790785221.907153458,9404,9414 +1790785222.013120015,9404,9414 +1790785222.120856681,9404,9414 +1790785222.236576779,9404,9414 +1790785222.333807345,9404,9414 +1790785222.462076277,9404,9414 +1790785222.563410519,9404,9414 +1790785222.674705317,9404,9414 +1790785222.781875436,9404,9414 +1790785222.888240027,9404,9414 +1790785223.013154620,9404,9414 +1790785223.113101197,9404,9414 +1790785223.229252020,9404,9414 +1790785223.330062507,9404,9414 +1790785223.433543294,9404,9414 +1790785223.546256088,9404,9414 +1790785223.657915379,9404,9414 +1790785223.767351295,9404,9414 +1790785223.876078382,9404,9414 +1790785223.981329080,9404,9414 +1790785224.104622585,9404,9414 +1790785224.213004569,9404,9414 +1790785224.324520788,9404,9414 +1790785224.436652445,9404,9414 +1790785224.539314983,9404,9414 +1790785224.653126260,9404,9624 +1790785224.757546848,9614,9624 +1790785224.871620316,8814,9624 +1790785224.975909163,9614,9624 +1790785225.080532224,9614,9624 +1790785225.191484787,9614,8824 +1790785225.294823663,9614,9624 +1790785225.413218872,9614,9624 +1790785225.515843249,9614,9624 +1790785225.621644953,9402,9624 +1790785225.735446929,9614,9624 +1790785225.836706261,9150,9160 +1790785225.960337970,9406,9416 +1790785226.058600157,9406,9416 +1790785226.167840991,9406,9416 +1790785226.274851779,9406,9416 +1790785226.376070820,9406,9416 +1790785226.499361145,9406,9416 +1790785226.598200910,9406,9416 +1790785226.695780869,9406,9416 +1790785226.812868793,9406,9416 +1790785226.925213262,9406,9416 +1790785227.033883739,9406,9416 +1790785227.134280221,9406,9416 +1790785227.233657431,9406,9416 +1790785227.356637983,9406,9416 +1790785227.464843325,9406,9416 +1790785227.570142523,9406,9416 +1790785227.681055376,9406,9416 +1790785227.789675752,9406,9416 +1790785227.905812575,9406,9416 +1790785228.004987183,9406,9416 +1790785228.131102820,9406,9416 +1790785228.237306459,9406,9416 +1790785228.347919678,9406,9416 +1790785228.453160026,9406,9416 +1790785228.561903764,9406,9416 +1790785228.672497593,9406,9416 +1790785228.775343593,9406,9416 +1790785228.878576067,9406,9416 +1790785228.993676178,9406,9416 +1790785229.095327705,9406,9416 +1790785229.229862738,9406,9416 +1790785229.344192490,9406,9416 +1790785229.450611152,9406,9416 +1790785229.560147179,9406,9416 +1790785229.661131758,9406,9416 +1790785229.784514054,9406,9416 +1790785229.884796974,9406,9416 +1790785230.000937807,9406,9416 +1790785230.107911895,9406,9416 +1790785230.209454960,9406,9416 +1790785230.321117721,9406,9416 +1790785230.433867225,9406,9416 +1790785230.551372683,9406,9416 +1790785230.661686499,9406,9416 +1790785230.772286018,9406,9416 +1790785230.893619601,9406,9416 +1790785231.000420086,9406,9416 +1790785231.122853211,9406,9416 +1790785231.225861183,9406,9416 +1790785231.338146052,9406,9416 +1790785231.449087335,9406,9416 +1790785231.553537753,9406,9416 +1790785231.677667018,9406,9416 +1790785231.784063519,9406,9416 +1790785231.911286219,9406,9416 +1790785232.023774639,9406,9416 +1790785232.132126833,9480,9490 +1790785232.240892971,9480,8824 +1790785232.343260056,9722,9732 +1790785232.458991784,9722,9732 +1790785232.563425602,9722,9732 +1790785232.676040714,9722,8896 +1790785232.780335841,9722,9732 +1790785232.888825837,9722,8824 +1790785233.009802544,9722,9732 +1790785233.113438523,9722,9732 +1790785233.226001035,9722,9732 +1790785233.331123822,9722,9732 +1790785233.431027548,9722,9732 +1790785233.544549601,9722,8824 +1790785233.648250191,8814,8824 +1790785233.761413850,8814,8824 +1790785233.859362213,8814,8824 +1790785233.957650982,8814,8824 +1790785234.067279409,8814,8824 +1790785234.167016614,8814,8824 +1790785234.274858491,8814,8824 +1790785234.388611216,8814,8824 +1790785234.496192271,8814,8824 +1790785234.606584498,8814,8824 +1790785234.716456939,8814,8824 +1790785234.829822210,8814,8824 +1790785234.937360024,8814,8824 +1790785235.043592553,8814,8824 +1790785235.162404426,8814,8824 +1790785235.271988034,8814,9372 +1790785235.393773611,9362,9628 +1790785235.508590919,9618,9628 +1790785235.627134759,9618,9628 +1790785235.740253277,9618,9628 +1790785235.888135684,9618,9628 +1790785236.021620746,9618,9628 +1790785236.128262980,9618,9628 +1790785236.247255795,9618,9628 +1790785236.368933841,9618,9628 +1790785236.465639651,9618,9628 +1790785236.567822963,9618,9628 +1790785236.661991214,9618,9628 +1790785236.765186268,9618,9628 +1790785236.862073070,9618,9628 +1790785236.956827358,9618,9628 +1790785237.060830071,9618,9628 +1790785237.169145804,9618,9628 +1790785237.273315079,9618,9628 +1790785237.380361368,9618,9628 +1790785237.477806526,9618,9628 +1790785237.589494578,9618,9628 +1790785237.685107865,9618,9628 +1790785237.803574234,9618,9628 +1790785237.902760512,9618,9628 +1790785238.000919179,9618,9628 +1790785238.117832360,9618,9628 +1790785238.215009726,9618,9628 +1790785238.310479971,9618,9628 +1790785238.413833277,9618,9628 +1790785238.514923807,9618,9628 +1790785238.634038234,9618,9628 +1790785238.740361984,9618,9628 +1790785238.863059842,9618,9628 +1790785238.967904125,9618,9628 +1790785239.081663940,9618,9628 +1790785239.188592358,9618,9628 +1790785239.301914838,9618,9628 +1790785239.417858678,9618,9628 +1790785239.526773038,9618,9628 +1790785239.632215008,9618,9628 +1790785239.751590018,9618,9628 +1790785239.855867485,9618,9628 +1790785239.985300520,9618,9628 +1790785240.093758205,9618,9628 +1790785240.208573052,9618,9628 +1790785240.320384905,9618,9628 +1790785240.421042980,9618,9628 +1790785240.533426350,9618,9628 +1790785240.639233214,9618,9628 +1790785240.765525153,9618,9628 +1790785240.873311330,9618,9628 +1790785240.991844880,9618,9628 +1790785241.101244506,9618,9628 +1790785241.205145038,9618,9628 +1790785241.331827062,9618,9628 +1790785241.433935773,9618,9628 +1790785241.560312963,9618,10076 +1790785241.671114525,10066,10076 +1790785241.781263039,10066,10076 +1790785241.892929841,10066,9784 +1790785241.993317633,8814,10410 +1790785242.103445766,10400,10410 +1790785242.205991973,10400,10410 +1790785242.310411571,10400,10410 +1790785242.413520054,8814,10424 +1790785242.515565915,10414,10424 +1790785242.633158575,10414,10424 +1790785242.735406168,10414,10424 +1790785242.850919814,8814,10424 +1790785242.956571196,10414,10424 +1790785243.060811192,10414,10424 +1790785243.181957173,10414,10424 +1790785243.279568723,9404,10424 +1790785243.380421180,10414,10424 +1790785243.484138060,10414,10424 +1790785243.584508252,10414,10424 +1790785243.692841185,8814,8824 +1790785243.793349758,10414,10424 +1790785243.895597262,10414,10424 +1790785244.001462626,10414,10424 +1790785244.101019569,10414,8824 +1790785244.211336545,10414,10424 +1790785244.311619806,10414,10424 +1790785244.416052294,10414,10424 +1790785244.520800366,10414,8824 +1790785244.639279786,10414,10424 +1790785244.749039635,10414,10424 +1790785244.926065467,10414,8824 +1790785245.039159084,10414,10424 +1790785245.137829047,10414,10424 +1790785245.235775081,10414,10424 +1790785245.338954614,10414,8824 +1790785245.447447080,9598,10424 +1790785245.558599205,10414,10424 +1790785245.659751776,10414,10424 +1790785245.759190467,10414,10424 +1790785245.859777931,8820,8830 +1790785245.956216428,8820,8830 +1790785246.060563066,8820,8830 +1790785246.154724266,8820,8830 +1790785246.250354723,8820,8830 +1790785246.355978035,8820,8830 +1790785246.450713612,8820,8830 +1790785246.545129356,8820,8830 +1790785246.644339744,8820,8830 +1790785246.743808846,8820,8830 +1790785246.850933385,8820,8830 +1790785246.945879085,8820,8830 +1790785247.043036530,8820,8830 +1790785247.143389911,8820,8830 +1790785247.243541700,8820,8830 +1790785247.340583144,8820,8830 +1790785247.445837192,8820,8830 +1790785247.543419761,8820,8830 +1790785247.645676515,8820,8830 +1790785247.741476084,8820,8830 +1790785247.839726382,8820,8830 +1790785247.953201264,8820,8830 +1790785248.050892725,8820,8830 +1790785248.157067833,8820,8830 +1790785248.257122422,8820,8830 +1790785248.356073937,8820,8830 +1790785248.460953940,8820,8830 +1790785248.563894472,8820,8830 +1790785248.672504438,8820,8830 +1790785248.771486294,8820,8830 +1790785248.871889036,8820,8830 +1790785248.981504924,8820,8830 +1790785249.076785937,8820,8830 +1790785249.179823370,8820,8830 +1790785249.293861389,8820,8830 +1790785249.396386295,8820,8830 +1790785249.505674649,8820,8830 +1790785249.613281404,8820,8830 +1790785249.716207285,8820,8830 +1790785249.823360064,8820,8830 +1790785249.927094545,8820,8830 +1790785250.037205649,8820,8830 +1790785250.140205690,8820,8830 +1790785250.241234040,8820,8830 +1790785250.344081110,8820,8830 +1790785250.443797254,8820,8830 +1790785250.548582047,8820,8830 +1790785250.649669286,8820,8830 +1790785250.748979836,8820,8830 +1790785250.864044366,8820,8830 +1790785250.963497028,8820,8830 +1790785251.067050486,8820,8830 +1790785251.173685890,8820,8830 +1790785251.273547366,8820,8830 +1790785251.383203854,8820,8830 +1790785251.483955310,8820,8830 +1790785251.589351030,8820,8830 +1790785251.695824082,8820,8830 +1790785251.795494265,8820,8830 +1790785251.908174418,8820,8830 +1790785252.012370084,8820,8830 +1790785252.113314042,8820,8830 +1790785252.214405842,8820,8830 +1790785252.316531424,8820,8830 +1790785252.426354104,8820,8830 +1790785252.527820839,8820,8830 +1790785252.630383166,8820,8830 +1790785252.739567779,8820,8830 +1790785252.837367261,8820,8830 +1790785252.945680694,8820,8830 +1790785253.044895173,8820,8830 +1790785253.146079644,8820,8830 +1790785253.247466367,8820,8830 +1790785253.349394667,8820,8830 +1790785253.456061161,8820,8830 +1790785253.553953894,8820,8830 +1790785253.652305363,8820,8830 +1790785253.764774643,8820,8830 +1790785253.864544458,8820,8830 +1790785253.969252920,8820,8830 +1790785254.071338801,8820,8830 +1790785254.174378864,8820,8830 +1790785254.296821419,8820,8830 +1790785254.400752511,8820,8830 +1790785254.510511761,8820,8830 +1790785254.610849292,8820,8830 +1790785254.713866904,8820,8830 +1790785254.835864474,8820,8830 +1790785254.934028731,8820,8830 +1790785255.037222455,8820,8830 +1790785255.142193930,8820,8830 +1790785255.240753121,8820,8830 +1790785255.349719091,8820,8830 +1790785255.449757999,8820,8830 +1790785255.551333605,8820,8830 +1790785255.657106598,8820,8830 +1790785255.757090325,8820,8830 +1790785255.862408664,8820,8830 +1790785255.961295389,8820,8830 +1790785256.061867713,8820,8830 +1790785256.169899813,8820,8830 +1790785256.272239767,8820,8830 +1790785256.379722381,8820,8830 +1790785256.478125360,8820,8830 +1790785256.580152580,8820,8830 +1790785256.692793863,8820,8830 +1790785256.794891654,8820,8830 +1790785256.917288439,8820,8830 +1790785257.027517614,8820,8830 +1790785257.130970631,8820,8830 +1790785257.248171566,8820,8830 +1790785257.350916955,8820,8830 +1790785257.460525193,8820,8830 +1790785257.562006308,8820,8830 +1790785257.665887349,8820,8830 +1790785257.771724914,8820,8830 +1790785257.872060735,8820,8830 +1790785257.985881552,8820,8830 +1790785258.081630320,8820,8830 +1790785258.182347386,8820,8830 +1790785258.288482634,8820,8830 +1790785258.389489913,8820,8830 +1790785258.499461955,8820,8830 +1790785258.593913419,8820,8830 +1790785258.696819720,8820,8830 +1790785258.801833595,8820,8830 +1790785258.901688461,8820,8830 +1790785259.007766698,8820,8830 +1790785259.103947722,8820,8830 +1790785259.203392203,8820,8830 +1790785259.317259410,8820,8830 +1790785259.414937831,8820,8830 +1790785259.512844304,8820,8830 +1790785259.615007746,8820,8830 +1790785259.719148201,8820,8830 +1790785259.826578374,8820,8830 +1790785259.927301340,8820,8830 +1790785260.033395808,8820,8830 +1790785260.137125478,8820,8830 +1790785260.238007085,8820,8830 +1790785260.353729813,8820,8830 +1790785260.459483137,8820,8830 +1790785260.571826146,8820,8830 +1790785260.680984659,8820,8830 +1790785260.788236819,8820,8830 +1790785260.907005672,8820,8830 +1790785261.015223424,8820,8830 +1790785261.127623604,8820,8830 +1790785261.297745076,8820,8830 +1790785261.415564548,8820,8830 +1790785261.520364261,8820,8830 +1790785261.628191128,8820,8830 +1790785261.744486302,8820,8830 +1790785261.845888756,8820,8830 +1790785261.953154587,8820,8830 +1790785262.052268214,8820,8830 +1790785262.150602353,8820,8830 +1790785262.256794292,8820,8830 +1790785262.361534013,8820,8830 +1790785262.473808162,8820,8830 +1790785262.575562779,8820,8830 +1790785262.679084747,8820,8830 +1790785262.783944900,8820,8830 +1790785262.888326709,8820,8830 +1790785262.996075825,8820,8830 +1790785263.101724047,8820,8830 +1790785263.201991118,8820,8830 +1790785263.307311346,8820,8830 +1790785263.410147036,8820,8830 +1790785263.520959658,8820,8830 +1790785263.627934706,8820,8830 +1790785263.745554245,8820,8830 +1790785263.854074610,8820,8830 +1790785263.961510223,8820,8830 +1790785264.081570861,8820,8830 +1790785264.189858244,8820,8830 +1790785264.310605409,8820,8830 +1790785264.419687801,8820,8830 +1790785264.529762385,8820,8830 +1790785264.641333934,8820,8830 +1790785264.749330284,8820,8830 +1790785264.868565652,8820,8830 +1790785264.977913227,8820,8830 +1790785265.090109954,8820,8830 +1790785265.200519451,8820,8830 +1790785265.310471243,8820,8830 +1790785265.428566869,8820,8830 +1790785265.537709901,8820,8830 +1790785265.653028894,8820,8830 +1790785265.760626309,8820,8830 +1790785265.867639797,8820,8830 +1790785265.984704010,8820,8830 +1790785266.094686142,8820,8830 +1790785266.209421069,8820,8830 +1790785266.318155777,8820,8830 +1790785266.426987535,8820,8830 +1790785266.547257056,8820,8830 +1790785266.657076346,8820,8830 +1790785266.776633067,8820,8830 +1790785266.889781446,8820,8830 +1790785267.005491694,8820,8830 +1790785267.111851644,8820,8830 +1790785267.212781172,8820,8830 +1790785267.326117843,8820,8830 +1790785267.428899123,8820,8830 +1790785267.541347753,8820,8830 +1790785267.646109675,8820,8830 +1790785267.750037548,8820,8830 +1790785267.861421446,8820,8830 +1790785267.963520698,8820,8830 +1790785268.084224193,8820,8830 +1790785268.189865635,8820,8830 +1790785268.301401885,8820,8830 +1790785268.411305206,8820,8830 +1790785268.515115307,8820,8830 +1790785268.621066633,8820,8830 +1790785268.719398542,8820,8830 +1790785268.823108892,8820,8830 +1790785268.935612513,8820,8830 +1790785269.037749615,8820,8830 +1790785269.137967665,8820,8830 +1790785269.238761051,8820,8830 +1790785269.340122255,8820,8830 +1790785269.449566671,8820,8830 +1790785269.552939947,8820,8830 +1790785269.686181016,8820,8830 +1790785269.785917330,8820,8830 +1790785269.971309538,8820,8830 +1790785270.073886775,8820,8830 +1790785270.177315521,8820,8830 +1790785270.276576540,8820,8830 +1790785270.379881786,8820,8830 +1790785270.480511631,8820,8830 +1790785270.592487296,8820,8830 +1790785270.703880824,8820,8830 +1790785270.815413254,8820,8830 +1790785270.923883759,8820,8830 +1790785271.032616976,8820,8830 +1790785271.139895287,8820,8830 +1790785271.247851516,8820,8830 +1790785271.355577202,8820,8830 +1790785271.465163480,8820,8830 +1790785271.571677592,8820,8830 +1790785271.681279200,8820,8830 +1790785271.793314305,8820,8830 +1790785271.903947395,8820,8830 +1790785272.010476718,8820,8830 +1790785272.114692583,8820,8830 +1790785272.220315856,8820,8830 +1790785272.328676139,8820,8830 +1790785272.433955537,8820,8830 +1790785272.534906476,8820,8830 +1790785272.639476076,8820,8830 +1790785272.752520893,8820,8830 +1790785272.863695479,8820,8830 +1790785272.973510669,8820,8830 +1790785273.085386532,8820,8830 +1790785273.191605102,8820,8830 +1790785273.297994882,8820,8830 +1790785273.403661005,8820,8830 +1790785273.510747854,8820,8830 +1790785273.618822914,8820,8830 +1790785273.726225036,8820,8830 +1790785273.836518102,8820,8830 +1790785273.941386405,8820,8830 +1790785274.046954937,8820,8830 +1790785274.156502154,8820,8830 +1790785274.265313102,8820,8830 +1790785274.369568959,8820,8830 +1790785274.477014452,8820,8830 +1790785274.585307105,8820,8830 +1790785274.697620093,8820,8830 +1790785274.807756937,8820,8830 +1790785274.923007860,8820,8830 +1790785275.035077856,8820,8830 +1790785275.147308234,8820,8830 +1790785275.259686053,8820,8830 +1790785275.365557518,8820,8830 +1790785275.470561713,8820,8830 +1790785275.579511153,8820,8830 +1790785275.693663173,8820,8830 +1790785275.803048388,8820,8830 +1790785275.914954693,8820,8830 +1790785276.020171410,8820,8830 +1790785276.125243416,8820,8830 +1790785276.232946471,8820,8830 +1790785276.343302238,8820,8830 +1790785276.447535074,8820,8830 +1790785276.553730473,8820,8830 +1790785276.658745378,8820,8830 +1790785276.760174502,8820,8830 +1790785276.869852820,8820,8830 +1790785276.977010210,8820,8830 +1790785277.083950567,8820,8830 +1790785277.199563984,8820,8830 +1790785277.311783911,8820,8830 +1790785277.423414952,8820,8830 +1790785277.532597766,8820,8830 +1790785277.640075958,8820,8830 +1790785277.760420950,8820,8830 +1790785277.874497149,8820,8830 +1790785277.986836858,8820,8830 +1790785278.097457667,8820,8830 +1790785278.213292796,8820,8830 +1790785278.320807400,8820,8830 +1790785278.429280285,8820,8830 +1790785278.535598445,8820,8830 +1790785278.638083521,8820,8830 +1790785278.748331606,8820,8830 +1790785278.855875350,8820,8830 +1790785278.964144413,8820,8830 +1790785279.071125810,8820,8830 +1790785279.179569595,8820,8830 +1790785279.284647860,8820,8830 +1790785279.403097220,8820,8830 +1790785279.513127063,8820,8830 +1790785279.617861364,8820,8830 +1790785279.725184476,8820,8830 +1790785279.832200384,8820,8830 +1790785279.939524656,8820,8830 +1790785280.045853456,8820,8830 +1790785280.147181889,8820,8830 +1790785280.248352400,8820,8830 +1790785280.351246410,8820,8830 +1790785280.464097745,8820,8830 +1790785280.578910433,8820,8830 +1790785280.693929843,8820,8830 +1790785280.808246384,8820,8830 +1790785280.923122883,8820,8830 +1790785281.038078802,8820,8830 +1790785281.150682704,8820,8830 +1790785281.257889554,8820,8830 +1790785281.365861133,8820,8830 +1790785281.473874593,8820,8830 +1790785281.586007849,8820,8830 +1790785281.701977210,8820,8830 +1790785281.820898075,8820,10410 +1790785281.933978942,10400,10410 +1790785282.044903425,10400,10410 +1790785282.156112232,10400,10410 +1790785282.268828215,10400,10410 +1790785282.379294723,10400,10410 +1790785282.492784245,10400,10410 +1790785282.602638835,10400,10410 +1790785282.714963184,10400,10410 +1790785282.824940766,10400,10410 +1790785282.933506412,10400,10410 +1790785283.037805589,8814,10410 +1790785283.140752951,10400,10410 +1790785283.247184422,10400,10410 +1790785283.353491242,10400,10410 +1790785283.454592472,8814,8824 +1790785283.558762817,10414,10424 +1790785283.666263111,10414,10424 +1790785283.772262757,10414,10424 +1790785283.876433062,10414,9026 +1790785283.987042962,10414,10424 +1790785284.090859253,10414,10424 +1790785284.199755363,10414,10424 +1790785284.305872370,10118,10424 +1790785284.411686015,10414,10424 +1790785284.520763956,10414,10424 +1790785284.625652880,10414,10424 +1790785284.733569638,8856,10424 +1790785284.838239619,10414,10424 +1790785284.944310377,10414,10424 +1790785285.050989481,10414,10424 +1790785285.154804462,8814,10424 +1790785285.263495339,10414,10424 +1790785285.369373384,10414,10424 +1790785285.478049992,10414,10424 +1790785285.580895712,10414,10424 +1790785285.685810866,10414,10424 +1790785285.791542149,10414,10424 +1790785285.896055318,10414,8824 +1790785286.006859640,10414,10424 +1790785286.112354010,10414,10424 +1790785286.219415569,10414,10424 +1790785286.319351376,10414,10424 +1790785286.430931126,10414,10424 +1790785286.537082454,10414,10424 +1790785286.640236508,10414,10424 +1790785286.741401619,8814,8824 +1790785286.848569768,10126,10424 +1790785286.955549866,10414,10424 +1790785287.060268848,10414,10424 +1790785287.161671192,10414,10424 +1790785287.270834714,10414,10424 +1790785287.376394775,10414,10424 +1790785287.481565462,10414,10424 +1790785287.588137145,10414,8824 +1790785287.692396652,10414,10424 +1790785287.797841492,10414,10424 +1790785287.903440744,10414,10424 +1790785288.007854002,10414,10424 +1790785288.115884471,10414,10424 +1790785288.221970889,10414,10424 +1790785288.327650761,10414,10424 +1790785288.431400402,8820,8866 +1790785288.537653652,10414,10424 +1790785288.641832017,10414,10424 +1790785288.747844773,10414,10424 +1790785288.853193602,10374,10424 +1790785288.957419358,10414,10424 +1790785289.061021597,10414,10424 +1790785289.165456445,10414,10424 +1790785289.275869652,9254,10424 +1790785289.384078014,10414,10424 +1790785289.489828968,10414,10424 +1790785289.592094171,10414,10424 +1790785289.700216192,8854,10424 +1790785289.809677168,10414,10424 +1790785289.912165134,10414,10424 +1790785290.019531826,10414,10424 +1790785290.120291382,8814,8824 +1790785290.225558421,10414,10424 +1790785290.336074119,10414,10424 +1790785290.441903083,10414,10424 +1790785290.545073107,10414,10424 +1790785290.648619255,10414,10424 +1790785290.757928170,10414,10424 +1790785290.859678957,10414,10424 +1790785290.971068056,9510,10424 +1790785291.075907898,10414,10424 +1790785291.181801874,10414,10424 +1790785291.289206026,10414,10424 +1790785291.392035296,8814,10424 +1790785291.500375799,10414,10424 +1790785291.607124944,10414,10424 +1790785291.708406646,10414,10424 +1790785291.809350305,8814,10424 +1790785291.914568292,10414,10424 +1790785292.028321898,10414,10424 +1790785292.133768077,10414,10424 +1790785292.239947376,8814,10424 +1790785292.346876863,10414,10424 +1790785292.449235328,10414,10424 +1790785292.555855461,10414,10424 +1790785292.661374162,9340,10424 +1790785292.768857956,10414,10424 +1790785292.873585357,10414,10424 +1790785292.980019169,10414,10424 +1790785293.084039082,10414,10424 +1790785293.187515750,10414,10424 +1790785293.290273589,10414,10424 +1790785293.393345112,10414,10424 +1790785293.497064762,8814,10424 +1790785293.604986630,10414,10424 +1790785293.706884950,10414,10424 +1790785293.813564414,10414,10424 +1790785293.919342478,10414,10424 +1790785294.026636749,10414,10424 +1790785294.126716107,10414,10424 +1790785294.229040372,10414,10424 +1790785294.333689662,10414,10424 +1790785294.436620973,10414,10424 +1790785294.543118486,10414,10424 +1790785294.644473129,10414,8826 +1790785294.751529717,10414,10424 +1790785294.857887707,10414,10424 +1790785294.964415330,10414,10424 +1790785295.067334071,10414,8824 +1790785295.169385502,8814,8824 +1790785295.276969617,8814,8824 +1790785295.388364185,8814,8824 +1790785295.499752523,8814,8824 +1790785295.605371965,8814,8824 +1790785295.712299572,8814,8824 +1790785295.816431008,8814,8824 +1790785295.925113624,8814,8824 +1790785296.033668440,8814,8824 +1790785296.143702753,8814,8824 +1790785296.245898425,8814,8824 +1790785296.343746818,8814,8824 +1790785296.452031321,8814,8824 +1790785296.557470981,8814,8824 +1790785296.664635241,8814,8824 +1790785296.771230464,8814,8824 +1790785296.877273291,8814,8824 +1790785296.985785866,8814,8824 +1790785297.094075338,8814,8824 +1790785297.207640412,8814,8824 +1790785297.313618838,8814,8824 +1790785297.423414868,8814,8824 +1790785297.532251097,8814,8824 +1790785297.642014636,8814,8824 +1790785297.748266996,8814,8824 +1790785297.854030149,8814,8824 +1790785297.963985541,8814,8824 +1790785298.071121530,8814,8824 +1790785298.179512774,8814,8824 +1790785298.288250232,8814,8824 +1790785298.395675415,8814,8824 +1790785298.501236956,8814,8824 +1790785298.607632117,8814,8824 +1790785298.716552077,8814,8824 +1790785298.825624539,8814,8824 +1790785298.945298122,8814,8824 +1790785299.050794282,8814,8824 +1790785299.156924790,8814,8824 +1790785299.262862946,8814,8824 +1790785299.371684724,8814,8824 +1790785299.480150779,8814,8824 +1790785299.586990596,8814,8824 +1790785299.696381441,8814,8824 +1790785299.805797256,8814,8824 +1790785299.915486055,8814,8824 +1790785300.024167982,8814,8824 +1790785300.133551788,8814,8824 +1790785300.241435915,8814,8824 +1790785300.348717796,8814,8824 +1790785300.457904000,8814,8824 +1790785300.564764075,8814,8824 +1790785300.672810465,8814,8824 +1790785300.781495663,8814,8824 +1790785300.890606085,8814,8824 +1790785300.997680163,8814,8824 +1790785301.104091575,8814,8824 +1790785301.212228735,8814,8824 +1790785301.320155224,8814,8824 +1790785301.427487586,8814,8824 +1790785301.533669064,8814,8824 +1790785301.640339948,8814,8824 +1790785301.749084806,8814,8824 +1790785301.856132434,8814,8824 +1790785301.968116549,8814,8824 +1790785302.076247540,8814,8824 +1790785302.185415873,8814,8824 +1790785302.292427981,8814,8824 +1790785302.402053219,8814,8824 +1790785302.510839128,8814,8824 +1790785302.618139839,8814,8824 +1790785302.721423544,8814,8824 +1790785302.827664313,8814,8824 +1790785302.933253025,8814,8824 +1790785303.041081052,8814,8824 +1790785303.149017070,8814,8824 +1790785303.257239313,8814,8824 +1790785303.365362064,8814,8824 +1790785303.473112460,8814,8824 +1790785303.582290893,8814,8824 +1790785303.688792985,8814,8824 +1790785303.796221557,8814,8824 +1790785303.905619313,8814,8824 +1790785304.013928236,8814,8824 +1790785304.121653302,8814,8824 +1790785304.233097040,8814,8824 +1790785304.340366982,8814,8824 +1790785304.460564221,8814,8824 +1790785304.567750261,8814,8824 +1790785304.735133012,0,2 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead_probe.py b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead_probe.py new file mode 100644 index 0000000..ab9082d --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/overhead_probe.py @@ -0,0 +1,37 @@ +"""GPU memory outside torch's allocator (whole card minus 2 MiB idle minus torch reserved), at quiet points +between loads. GPU 3 carries only this process.""" +import json, subprocess, sys, threading, time, urllib.request +URL = "http://127.0.0.1:18033" +TOKEN = open("token").read().strip() +H = {"Authorization": "Bearer " + TOKEN, "Content-Type": "application/json"} +def card(): + return int(subprocess.check_output(["nvidia-smi", "-i", "3", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]).strip()) +def health(): + return json.load(urllib.request.urlopen(URL + "/health"))["model"] +def post(path, body): + r = urllib.request.Request(URL + path, data=json.dumps(body).encode(), headers=H) + try: + return urllib.request.urlopen(r).status + except urllib.error.HTTPError as e: + return e.code +SMALL = {"id": "x", "state": "The deploy passed.", "question": "Did it pass?", + "options": [{"id": "yes", "description": "Yes"}, {"id": "no", "description": "No"}]} +out = [] +def snap(tag): + time.sleep(1.5) + c, h = card(), health() + r = h["reserved_gib"] * 1024 + row = {"tag": tag, "card_mib": c, "reserved_mib": round(r), "outside_mib": round(c - 2 - r), "max_reserved_mib": round(h["max_reserved_gib"] * 1024)} + out.append(row); print(row, flush=True) +snap("rest") +for burst in range(2): + ts = [threading.Thread(target=post, args=("/decide", SMALL)) for _ in range(30)] + [t.start() for t in ts]; [t.join() for t in ts] + snap(f"after concurrent burst {burst + 1}") +subprocess.run([sys.executable, "bench_shape.py", "--backend", "semif", "--url", URL, "--token-file", "token", + "--long-state-file", "long_state.txt", "--label", "probe", "--out", sys.argv[1] + "/shape.json", "--runs", "1", "--per", "5"], check=True) +snap("after latency shapes") +subprocess.run([sys.executable, "checks.py", "--url", URL, "--token-file", "token", "--out", sys.argv[1] + "/checks.json", + "--checks", "maxreq,oom", "--repeats", "2"], check=True) +snap("after largest requests") +json.dump(out, open(sys.argv[1] + "/overhead.json", "w"), indent=1) diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/poll.sh b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/poll.sh new file mode 100755 index 0000000..4f076e3 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/poll.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# poll.sh : unix time, that container's process GPU memory (MiB, nvidia-smi per process), card total +pid=$(docker inspect -f '{{.State.Pid}}' "$1") +gpu=$(nvidia-smi --query-compute-apps=pid,gpu_uuid --format=csv,noheader | awk -F', ' -v p="$pid" '$1==p{print $2}') +echo "# pid=$pid gpu=$gpu" > "$2" +while [ -d /proc/$pid ]; do + procmem=$(nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits | awk -F', ' -v p="$pid" '$1==p{print $2}') + card=$(nvidia-smi --id="$gpu" --query-gpu=memory.used --format=csv,noheader,nounits) + printf '%s,%s,%s\n' "$(date +%s.%N)" "${procmem:-0}" "$card" >> "$2" +done diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/container.id b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/container.id new file mode 100644 index 0000000..1a2b9bf --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/container.id @@ -0,0 +1 @@ +1d98f12c909d19f7e157823e29412e4fb44aa22db94c6ee4a96d8453aed29c70 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/events.txt b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/events.txt new file mode 100644 index 0000000..20e4a3e --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/events.txt @@ -0,0 +1,3 @@ +1790785010.413756011 sets:start +1790785010.470560236 sets:done +gpu3 after: 2 MiB diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/health-end.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/health-end.json new file mode 100644 index 0000000..37aac99 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/health-end.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.969,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/health-rest.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/health-rest.json new file mode 100644 index 0000000..37aac99 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/health-rest.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.969,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/vram.csv new file mode 100644 index 0000000..1902bc6 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/probe/vram.csv @@ -0,0 +1,28 @@ +# pid=1840754 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790785007.592863455,8812,8822 +1790785007.701879593,8812,8822 +1790785007.808902409,8812,8822 +1790785007.915876585,8812,8822 +1790785008.021279853,8812,8822 +1790785008.129614494,8812,8822 +1790785008.237274588,8812,8822 +1790785008.347694802,8812,8822 +1790785008.453754747,8812,8822 +1790785008.563765228,8812,8822 +1790785008.672580344,8812,8822 +1790785008.781543962,8812,8822 +1790785008.889253036,8812,8822 +1790785008.995532764,8812,8822 +1790785009.100480397,8812,8822 +1790785009.209746018,8812,8822 +1790785009.316264048,8812,8822 +1790785009.423762660,8812,8822 +1790785009.532005020,8812,8822 +1790785009.643907242,8812,8822 +1790785009.752750529,8812,8822 +1790785009.861883069,8812,8822 +1790785009.969250889,8812,8822 +1790785010.076693700,8812,8822 +1790785010.185418326,8812,8822 +1790785010.293297732,8812,8822 +1790785010.399472568,8812,8822 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/checks.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/checks.json new file mode 100644 index 0000000..f631298 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/checks.json @@ -0,0 +1,476 @@ +{ + "url": "http://127.0.0.1:18033", + "started_utc": "2026-09-30T16:14:47Z", + "health": { + "status": "ok", + "model": { + "name": "Intern-Decision-4B", + "source": "internlm/Intern-Decision-4B", + "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "checkpoint": "/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "inference_py_sha256": "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863", + "temperature": 1.99241824, + "dtype": "bfloat16", + "attn_implementation": "sdpa", + "device": "cuda", + "max_length": 8192, + "torch_version": "2.10.0+cu128", + "transformers_version": "5.17.0", + "vision_tower": "removed", + "device_name": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", + "allocated_gib": 7.944, + "reserved_gib": 7.969, + "max_reserved_gib": 8.861 + }, + "vram_cap_gib": 9.25, + "max_tokens": 8192, + "max_decisions": 64, + "max_questions_per_call": 16, + "chunking": "/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.", + "workloads": [] + }, + "auth": { + "/decide": { + "no_token": 401, + "wrong_token": 401, + "right_token": 200 + }, + "/decide/shared": { + "no_token": 401, + "wrong_token": 401, + "right_token": 200 + }, + "health_no_token": 200, + "pass": true, + "t_start": 1790784887.4892507, + "t_end": 1790784887.5748613 + }, + "chunking": { + "status": [ + 200, + 200, + 200 + ], + "timing": { + "total_seconds": 0.1498984640929848, + "batch_size": 20, + "calls": 2, + "questions_per_call": [ + 16, + 4 + ], + "input_tokens": [ + 1884, + 1328 + ], + "inference_seconds": 0.1373 + }, + "rows": [ + { + "id": "d0", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q1", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d1", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q2", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d2", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q3", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d3", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q4", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d4", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q5", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d5", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q6", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d6", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q7", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d7", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q8", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d8", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q9", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d9", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q10", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d10", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q11", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d11", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q12", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d12", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q13", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d13", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q14", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d14", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q15", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d15", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q16", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d16", + "top_whole": "unclear", + "top_split": "unclear", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q1", + "questions": 4 + }, + "sha_equal": true + }, + { + "id": "d17", + "top_whole": "unclear", + "top_split": "unclear", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q2", + "questions": 4 + }, + "sha_equal": true + }, + { + "id": "d18", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q3", + "questions": 4 + }, + "sha_equal": true + }, + { + "id": "d19", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q4", + "questions": 4 + }, + "sha_equal": true + } + ], + "e2e_ms": 157.0, + "pass": true, + "t_start": 1790784887.5748994, + "t_end": 1790784887.894067 + }, + "queue": { + "sent": 48, + "max_queue": 32, + "ok": 32, + "busy_429": 16, + "codes_429": [ + "busy" + ], + "other": [], + "pass": true, + "t_start": 1790784887.8941243, + "t_end": 1790784895.1853623 + }, + "fit": { + "largest_ok_state_chars": 5514, + "largest_ok_tokens": 6826, + "probes": [ + [ + 31950, + 422, + null + ], + [ + 16475, + 422, + null + ], + [ + 8737, + 503, + null + ], + [ + 4868, + 200, + 6692 + ], + [ + 6802, + 503, + null + ], + [ + 5835, + 503, + null + ], + [ + 5351, + 200, + 6785 + ], + [ + 5593, + 503, + null + ], + [ + 5472, + 200, + 6818 + ], + [ + 5532, + 503, + null + ], + [ + 5502, + 200, + 6824 + ], + [ + 5517, + 503, + null + ], + [ + 5509, + 200, + 6825 + ], + [ + 5513, + 200, + 6826 + ], + [ + 5515, + 503, + null + ], + [ + 5514, + 200, + 6826 + ] + ], + "t_start": 1790784895.1854463, + "t_end": 1790784905.1581368 + }, + "maxreq": { + "state_chars": 11337, + "tokens_per_call": null, + "decisions": 64, + "options_each": 16, + "body_bytes": 100953, + "runs": [ + { + "status": 503, + "e2e_ms": 188.3, + "code": "out_of_memory", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 8.262, + "max_reserved_gib": 9.248, + "t_end": 1790784906.9150267 + }, + { + "status": 503, + "e2e_ms": 186.1, + "code": "out_of_memory", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 8.262, + "max_reserved_gib": 9.248, + "t_end": 1790784907.104969 + }, + { + "status": 503, + "e2e_ms": 198.2, + "code": "out_of_memory", + "calls": null, + "input_tokens": null, + "reserved_gib_after": 8.262, + "max_reserved_gib": 9.248, + "t_end": 1790784907.306796 + } + ], + "t_start": 1790784905.1582026, + "t_end": 1790784907.3068023 + }, + "oom": { + "oom_status": 503, + "oom_error": { + "code": "out_of_memory", + "message": "CUDA out of memory. Tried to allocate 64.00 MiB. GPU 0 has a total capacity of 94.97 GiB of which 84.85 GiB is free. Including non-PyTorch memory, this process has 10.11 GiB memory in use. 9.25 GiB allowed; Of the allocated memory 9.12 GiB is allocated by PyTorch, and 97.33 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)" + }, + "oom_ms": 197.6, + "reserved_gib": { + "before": 8.262, + "after_oom": 8.262, + "after_next_request": 8.287 + }, + "next_request_status": 200, + "next_top": "yes", + "pass": true, + "t_start": 1790784907.3068488, + "t_end": 1790784907.554095 + }, + "finished_utc": "2026-09-30T16:15:07Z" +} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/container.id b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/container.id new file mode 100644 index 0000000..2ce9b32 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/container.id @@ -0,0 +1 @@ +35bbfe41a4b1b3227c24caa36199780398e09966ceb273fbe9402fb1631d6782 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/events.txt b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/events.txt new file mode 100644 index 0000000..65ea122 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/events.txt @@ -0,0 +1,5 @@ +1790784749.283255359 sets:start +1790784842.311562084 sets:done +1790784887.437615498 shape:done +1790784907.572533137 checks:done +gpu3 after: 2 MiB diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/health-end.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/health-end.json new file mode 100644 index 0000000..1281e0f --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/health-end.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":8.19,"reserved_gib":8.287,"max_reserved_gib":9.248},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/health-rest.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/health-rest.json new file mode 100644 index 0000000..37aac99 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/health-rest.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.969,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/sets.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/sets.json new file mode 100644 index 0000000..9785f70 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/sets.json @@ -0,0 +1 @@ +{"label": "r1", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:12:29Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 49.40379201434553, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 37.78}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25763410329819, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.04}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.679856944829226, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 33.86}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70842689462006, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 33.86}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8461690377444, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 33.97}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05454105511308, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.21}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67094698175788, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 33.95}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20988309942186, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.43}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31136303395033, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.25}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97045100852847, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.15}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20351308584213, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.46}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81543904915452, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 34.93}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99107002466917, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.29}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23890311084688, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.45}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04749094322324, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 34.2}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.106171948835254, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.41}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30925405398011, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.52}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09713090211153, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.28}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.816388834267855, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.19}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.990819964557886, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.25}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26495406590402, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.41}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29887306690216, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.61}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.210093112662435, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.44}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.564717000350356, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.71}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35613403841853, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.66}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.398806074634194, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.58}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.272894056513906, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.36}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.860758904367685, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.0}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.87856903485954, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.18}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.11481408774853, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.08}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.846619099378586, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.13}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.155193043872714, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.36}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19582292251289, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.33}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.261202931404114, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.51}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1894130948931, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.37}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.479507038369775, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.55}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13588113896549, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.3}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47106599621475, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.67}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19600290060043, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.3}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99159994721413, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.31}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19108295626938, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.4}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25841408595443, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.39}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89255891367793, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.18}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31220401823521, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.49}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03119093552232, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 34.14}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86112887226045, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.02}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77005786448717, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 33.99}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.018420873209834, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.11}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.598207127302885, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.62}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89333889633417, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.01}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.509874975308776, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 33.65}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03270107507706, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.27}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.969071021303535, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.17}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.814798997715116, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 35.11}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99702094309032, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.28}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.289513973519206, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.54}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.148181116208434, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.39}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.381924921646714, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.46}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.24018384329975, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 35.34}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27301396429539, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.24}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87914996407926, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.15}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44288510642946, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.57}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84312920086086, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.01}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16825204901397, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.32}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01794100925326, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.3}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34130505472422, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.49}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06395113840699, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.28}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22626296803355, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.48}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9603110011667, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.27}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.199222948402166, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.29}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81758884340525, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.1}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0145000051707, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.27}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.159222178161144, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.3}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19488298520446, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.47}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92959900945425, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.16}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76221885345876, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.9}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13904204778373, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 34.42}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3486140742898, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.57}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.485294975340366, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.54}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03571087680757, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.23}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53924695774913, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.66}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21053316257894, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.4}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71735688112676, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 33.97}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.547976011410356, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.7}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22489299252629, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.41}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.144392961636186, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.4}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11102204397321, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.3}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.91638114489615, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 35.18}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67753788270056, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.94}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29538409970701, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.46}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.005001213401556, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.25}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62918710708618, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 34.81}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.167513044551015, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.34}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.133551901206374, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.36}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92078893072903, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 33.96}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98795987293124, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.18}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.004890851676464, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.2}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71234909072518, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.79}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10566204972565, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.35}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01149998232722, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.15}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40236512385309, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.58}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56842692382634, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.74}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.663735983893275, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 33.94}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.863869056105614, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.01}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.720747128129005, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 33.88}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05381113104522, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.35}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.50525398552418, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 33.78}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25800383836031, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.39}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83882812038064, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.01}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.730676889419556, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.05}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78287798538804, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 33.93}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93223907612264, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 33.99}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.542975110933185, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 33.7}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84639907442033, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.16}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.611316027119756, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 33.88}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92154982127249, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.03}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75463702529669, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 33.91}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01544110663235, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.29}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75259789451957, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.01}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14820207096636, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.22}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.544434959068894, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 33.71}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88173904083669, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.03}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.584296030923724, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 33.81}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00520982965827, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.19}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75950807519257, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 33.97}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81806894019246, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.08}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65186697617173, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 33.96}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72904800623655, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 33.99}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73249715939164, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 33.74}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05915086902678, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.1}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83467791415751, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.03}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69651714153588, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.17}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.50253498926759, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 33.96}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68858692981303, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.01}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53887496329844, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 33.78}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93772992119193, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.09}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52986488305032, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 33.75}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94207011722028, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.2}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.454143933951855, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 33.74}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75586706958711, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 33.95}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.667006788775325, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 33.92}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68222716078162, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 33.9}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59108513966203, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 33.82}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86400898732245, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.09}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.50415502488613, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 33.74}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63938585668802, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 33.89}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.078430876135826, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.18}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66806686669588, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 33.93}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.435354966670275, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 33.69}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.875498946756124, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 34.12}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.38915391638875, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 33.58}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.974790040403605, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.21}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60272597335279, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 33.82}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8648290168494, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.13}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81785799562931, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.03}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.029941100627184, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 34.21}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.37123294174671, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 33.6}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9171891361475, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.13}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59701603464782, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 33.76}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82801895774901, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.06}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.426463862881064, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 33.65}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89411887899041, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 34.15}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.54378606006503, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 33.8}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67567693628371, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 33.91}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63362585939467, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 33.8}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.842438858002424, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.12}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.392423091456294, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 33.6}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96650988422334, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.18}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48598399013281, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 33.69}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64185595698655, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 33.88}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.630015820264816, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 33.9}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.828848998993635, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.01}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.45776421763003, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 33.69}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32333402149379, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 34.52}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5298050455749, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 33.78}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.831069039180875, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.09}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88796889036894, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 34.09}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.683427169919014, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 33.93}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57010500319302, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 33.79}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85618890449405, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 34.12}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.43160406500101, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 33.68}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92549001425505, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 34.19}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.608156049624085, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 33.77}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.791868042200804, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.01}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60737699829042, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 33.9}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84285911731422, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 34.09}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.490614991635084, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 33.7}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89041989855468, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 34.09}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.484044045209885, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 33.73}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92783904261887, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.18}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.35678417049348, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 33.62}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83338896371424, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.06}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70035698451102, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 33.89}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.727837985381484, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 33.99}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60180582664907, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 33.81}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88601893372834, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.11}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77165787667036, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 33.89}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91367898136377, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 34.19}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56906494870782, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 33.82}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.761077819392085, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 34.0}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74990707077086, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 33.98}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84055805206299, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.06}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.494305124506354, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 33.79}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58706711232662, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 34.83}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.55042499303818, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.79}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84410895220935, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.07}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68486699461937, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 33.91}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.705508129671216, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 33.96}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48175508715212, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 33.72}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.425975082442164, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 34.55}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.602245876565576, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 33.84}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90755018033087, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 34.19}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21827291883528, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 34.38}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65737621486187, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 33.94}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60892602056265, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 33.88}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93464002572, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 34.16}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56154521740973, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 33.8}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9989899918437, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 34.2}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.58338589593768, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 33.86}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88305011019111, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.16}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.41791408881545, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 33.67}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00516000390053, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 34.16}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.786607932299376, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 33.97}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78741701878607, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.04}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94724012166262, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 34.16}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.968820029869676, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.2}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.591595970094204, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 33.86}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06731181591749, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 34.31}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.715226946398616, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 33.97}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91527992486954, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.09}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0707509573549, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 34.18}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.718887044116855, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 33.99}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.373223178088665, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 33.66}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17315196990967, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 34.36}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.440074210986495, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 33.71}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81609802879393, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 34.1}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82237793691456, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 34.11}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.940769059583545, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 34.15}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48045496083796, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 33.72}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00272017158568, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 34.15}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.54131502844393, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 33.8}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87669895589352, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 34.18}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.58092604018748, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 33.81}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80822905153036, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.04}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.775737069547176, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 33.95}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77542810328305, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.02}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48144495673478, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 33.74}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93750989623368, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 34.19}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85455908998847, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 34.11}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87534900382161, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.13}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52753587812185, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 33.78}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.982620833441615, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.2}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43330598808825, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848489195108414, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.12}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94847016595304, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 34.12}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08705096505582, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.19}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93775902502239, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 34.06}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.930149886757135, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.04}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.876219091936946, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 33.92}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88575909473002, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.03}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.112632067874074, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 34.2}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2208128683269, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.2}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.83066011965275, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 34.92}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.724328925833106, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.87}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01912984251976, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.002841010689735, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.14}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.997390910983086, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 34.09}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95686999708414, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.1}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07372210919857, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 34.18}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97678912431002, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.1}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08611195906997, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 34.11}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89635009504855, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.05}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.858647828921676, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 33.93}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.717907059937716, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.84}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81351896375418, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 33.84}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10834099911153, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.2}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76584782078862, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 33.95}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76126780919731, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.05}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.932109855115414, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.823387956246734, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.08}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.881449865177274, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 34.12}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77712706848979, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.0}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49940707348287, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 34.75}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51520696096122, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.78}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.518606055527925, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 34.72}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64880797825754, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.91}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85573993809521, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 35.07}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37095417454839, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.63}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.505116080865264, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 34.78}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51929593272507, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.77}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.368194898590446, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 34.65}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78161813877523, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.97}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.916018929332495, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75588802434504, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.99}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06287080422044, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 34.09}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.862948909401894, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.97}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.801499081775546, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 33.97}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80392797105014, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.93}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11071121506393, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.001210963353515, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.22}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8335279636085, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 34.1}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88962897285819, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.15}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72535717487335, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 33.97}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92285902798176, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.06}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.104851100593805, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 34.2}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.989900982007384, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.12}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24966291338205, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75445797853172, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.9}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.000761134549975, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 34.16}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97770903818309, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.1}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98318009264767, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 34.06}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.736177979037166, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.9}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96110984310508, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 34.07}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.728106904774904, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.89}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.209441885352135, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 34.33}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69085702858865, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.84}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92784905433655, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 33.99}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.855089012533426, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.99}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77413798868656, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 33.9}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62601602263749, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.8}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.761197028681636, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 33.92}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79564711637795, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.88}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68497712351382, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 33.77}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04330208711326, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72475693747401, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 33.81}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84174898080528, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.94}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5985759999603, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 33.66}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.677227122709155, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.83}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82057908736169, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 33.95}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.55273490585387, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.72}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.541485926136374, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 33.6}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.850868025794625, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.96}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.588766146451235, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 33.68}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75998677499592, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.94}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79540893062949, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 33.87}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77324904128909, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81421815417707, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62476688623428, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.77}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57019603997469, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 33.62}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87766008079052, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.97}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72022605314851, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 33.83}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.587066017091274, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.74}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.624886095523834, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 33.75}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68317594565451, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.81}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94829996675253, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 34.04}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64565598592162, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.76}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52869584038854, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 33.62}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.082551047205925, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.2}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7158071603626, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 33.77}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64900595322251, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.82}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.970950197428465, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.703266901895404, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60395601764321, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 33.66}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78644821047783, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56585584394634, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 33.69}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.802228074520826, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53225512243807, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 33.65}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91357001662254, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.02}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68876690790057, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 33.85}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.743028089404106, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.550624994561076, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 33.67}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0811210013926, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.19}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.605496192350984, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 33.68}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.529756151139736, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.68}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.713796900585294, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 33.85}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72150708734989, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.85}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73861804418266, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 33.73}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.727446829900146, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.91}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.021331023424864, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84644913300872, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.97}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68369702063501, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 33.77}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.866829032078385, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.01}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87384887598455, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.696785828098655, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.85}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5882960613817, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 33.66}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70758684165776, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.792878061532974, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.893788957968354, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.01}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6048461291939, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 33.66}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73049691133201, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.83}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.800567992031574, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 33.9}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69554693624377, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67233698070049, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 33.69}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05010004341602, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.13}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.643916042521596, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 33.79}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92060010507703, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57555510289967, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 33.62}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80516895838082, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.89}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71681694686413, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 33.88}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68662696145475, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.84}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.49801481515169, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 33.61}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.856209160760045, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.93}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.89734291099012, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 36.84}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80771798826754, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.92}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.958502070978284, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 34.97}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97406989522278, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.05}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62270696461201, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 34.66}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79294791072607, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 33.92}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73618885315955, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 34.7}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14432194828987, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.24}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6506180036813, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 34.62}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86813893727958, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.02}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30843402445316, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 34.46}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76326689310372, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.0}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18827199190855, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 34.33}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78716788999736, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.02}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19653282314539, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 34.33}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99327097274363, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.21}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3752738814801, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 34.5}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81175806559622, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.09}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.530126981437206, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 34.67}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70250694267452, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 33.98}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07526088692248, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 34.29}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74910683184862, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.0}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47324598953128, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 34.61}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.938310135155916, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.24}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57034591399133, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 34.7}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.758908070623875, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.04}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77001891285181, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 34.77}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9586399756372, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.06}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28933399543166, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 34.45}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68887703679502, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 33.97}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54565608687699, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 34.65}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.925410153344274, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.18}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61878702789545, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 34.58}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69030708260834, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.95}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62414795346558, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 34.8}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.61312581971288, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 33.89}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.225082982331514, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.844728980213404, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.02}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68935799598694, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 34.87}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848668940365314, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.12}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48201602138579, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 34.65}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63533600047231, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.93}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.662437884137034, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 34.8}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.642675986513495, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 33.88}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1958629693836, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69373714365065, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 33.99}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51710616052151, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 34.61}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88624897040427, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.17}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30067401193082, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 34.47}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7486581671983, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.98}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48190705664456, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 34.65}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.957580829039216, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.24}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.864327965304255, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 34.04}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.769628059118986, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.04}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44201501831412, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 34.62}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03259094618261, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.29}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.459246115759015, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 34.63}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6029751021415, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.85}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64956793747842, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 34.71}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.565635818988085, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 33.84}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65465894341469, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 35.88}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43282495997846, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.54}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.203004049137235, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 35.83}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.82157799974084, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.18}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.21909208595753, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 34.42}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.74896603822708, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.05}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.20120091363788, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 34.43}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.57775402441621, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 33.76}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.955038852989674, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 34.05}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.718895960599184, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 33.9}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.0423200763762, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 34.16}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.62559396959841, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 33.77}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.666467087343335, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 34.77}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.44755303300917, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.59}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.246311850845814, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 34.36}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.34424204938114, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.13}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2912441380322, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 34.23}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62576503120363, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 33.72}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.25686499476433, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 35.25}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.867909133434296, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.01}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70751808770001, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 34.64}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88866016454995, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.84}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29177382774651, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 34.29}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64670581929386, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 33.75}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42870595306158, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 34.34}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67938685975969, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 33.82}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60421695187688, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 34.6}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.11295400373638, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.1}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45888499543071, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 34.43}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.396175088360906, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.46}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4073451384902, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 34.46}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90187889151275, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 33.97}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71265898831189, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 34.61}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48630592599511, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.59}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.12282299622893, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 35.03}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48133615590632, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.59}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.232354912906885, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.12}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.679737037047744, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.81}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53428580239415, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 34.4}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03741100616753, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.11}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49240608885884, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 34.5}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78887686692178, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 33.84}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.639067973941565, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.623816937208176, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 33.74}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7522490452975, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 34.74}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.995479837059975, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.01}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76762913912535, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 34.71}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13654191233218, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.13}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.311163963750005, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 34.38}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.034469889476895, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.11}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.743528839200735, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 34.68}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.517335964366794, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.44987694174051, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 35.36}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.522616097703576, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.61}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.162834011018276, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 35.12}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62807697057724, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.75}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.12351287342608, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.05}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55078720860183, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.6}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39984582737088, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.34}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10129181854427, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.09}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.30092493444681, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 35.25}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39962517283857, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.54}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01834210753441, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 34.93}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92632091790438, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.01}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.886461010202765, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 34.78}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94677003659308, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.07}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.909939885139465, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 34.81}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93944006226957, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.03}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.627527955919504, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65116794966161, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.69}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.99640201032162, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.03}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93051093816757, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.1}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.96616103500128, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 34.91}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57336689159274, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.66}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.930960999801755, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 34.88}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90867917612195, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.07}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.323854165151715, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 34.3}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.762359948828816, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.83}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.813160102814436, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 34.84}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.726548966020346, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.8}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.674698047339916, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 34.69}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86570003628731, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 33.96}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13752415589988, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 35.06}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38018497824669, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.49}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.1894139572978, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 35.05}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.592136999592185, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.68}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.18698390386999, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 35.1}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.99368184991181, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.14}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13718305900693, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 35.09}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82558999955654, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.84}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.19281398318708, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 35.26}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27798396721482, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.32}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57403697818518, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 34.54}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99628007598221, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.06}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86425106227398, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 34.76}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87078901566565, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 33.93}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4946859665215, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 34.54}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22243313677609, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.3}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7083090133965, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 34.68}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92654985934496, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 33.98}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3543337918818, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 34.4}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.972640082240105, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.07}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53054700791836, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 34.46}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02632011845708, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.19}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.095281993970275, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 34.2}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75091709010303, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 33.82}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.18772407248616, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.3}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67932811751962, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.88}], "seconds": 20.8, "failures": 0}, "rotations": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.4425510596484, "probs": [0.021278230729761302, 0.041549605758459535, 0.9371721635117792], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.82366502098739, "probs": [0.9569751557868662, 0.022187022115204392, 0.020837822097929372], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.76933000609279, "probs": [0.028764030986406904, 0.9453276881980787, 0.025908280815514436], "top": "supported", "agreement": 1.0, "input_tokens": 196}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.65763905085623, "probs": [0.9253638152569406, 0.02190751448970375, 0.052728670253355744], "top": "insufficient", "agreement": 1.0, "input_tokens": 196}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.66764005832374, "probs": [0.9597742641946788, 0.021791404944545737, 0.01843433086077556], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17288604751229, "probs": [0.0761485956363141, 0.025667636008847486, 0.8981837683548385], "top": "contradicted", "agreement": 1.0, "input_tokens": 206}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.8298510722816, "probs": [0.9159676813214028, 0.0555726559991739, 0.028459662679423257], "top": "contradicted", "agreement": 1.0, "input_tokens": 208}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.77833210118115, "probs": [0.02148731809580675, 0.05171731055352475, 0.9267953713506685], "top": "insufficient", "agreement": 1.0, "input_tokens": 200}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.96646212413907, "probs": [0.025695874936561842, 0.016912927133118746, 0.9573911979303193], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.14944708161056, "probs": [0.9650011484904946, 0.014421131045601873, 0.02057772046390364], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6484910659492, "probs": [0.9685134503822489, 0.011261347946432, 0.020225201671319158], "top": "supported", "agreement": 1.0, "input_tokens": 193}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84928102232516, "probs": [0.015601263068851802, 0.9601947660643307, 0.024203970866817515], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13548506423831, "probs": [0.008885594588144235, 0.029266695521859745, 0.9618477098899961], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.44721984863281, "probs": [0.0467350628868348, 0.02343846611289458, 0.9298264710002706], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.90515292622149, "probs": [0.02661110718913202, 0.02251153411335893, 0.950877358697509], "top": "contradicted", "agreement": 1.0, "input_tokens": 199}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.09594679065049, "probs": [0.9629993612353499, 0.018113485593456775, 0.01888715317119321], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.10172704420984, "probs": [0.8820069055961203, 0.07635736188763995, 0.04163573251623969], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41804903186858, "probs": [0.02701681560472035, 0.9453954270569487, 0.027587757338330996], "top": "supported", "agreement": 1.0, "input_tokens": 198}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07355397567153, "probs": [0.02293683426823875, 0.9487912139983355, 0.028271951733425657], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21206608973444, "probs": [0.019367602097265263, 0.013573064959569787, 0.9670593329431649], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1547050010413, "probs": [0.03987925527749218, 0.02220468207692111, 0.9379160626455868], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.6701290179044, "probs": [0.02674495094579214, 0.9358820879397224, 0.03737296111448533], "top": "contradicted", "agreement": 1.0, "input_tokens": 211}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41344899684191, "probs": [0.03051840202788244, 0.9419931233693989, 0.027488474602718756], "top": "contradicted", "agreement": 1.0, "input_tokens": 201}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06251291371882, "probs": [0.9573434428870992, 0.022664616232544885, 0.019991940880355972], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17374495975673, "probs": [0.20778523836492876, 0.6164398690397396, 0.17577489259533158], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21760489791632, "probs": [0.01494019219967109, 0.958782768824845, 0.026277038975483776], "top": "permitted", "agreement": 1.0, "input_tokens": 218}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.01784498430789, "probs": [0.038717451842913995, 0.06953592628537188, 0.8917466218717142], "top": "permitted", "agreement": 1.0, "input_tokens": 203}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25666596367955, "probs": [0.8769652162510786, 0.06982844563770668, 0.053206338111214745], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.88328197970986, "probs": [0.8731504724710493, 0.09124479732837286, 0.03560473020057783], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21535598672926, "probs": [0.03149783270455631, 0.9131032971636024, 0.055398870131841214], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.46503688581288, "probs": [0.1973515606103041, 0.4555434753670417, 0.3471049640226543], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 209}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.83768093399704, "probs": [0.02274552846853418, 0.9023369803119639, 0.07491749121950192], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.83596217818558, "probs": [0.09302877372922651, 0.07088401170776323, 0.8360872145630103], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1261539105326, "probs": [0.8635450750116728, 0.09024102515984218, 0.04621389982848493], "top": "permitted", "agreement": 1.0, "input_tokens": 211}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.51848891191185, "probs": [0.028891977481104857, 0.0412263798327744, 0.9298816426861207], "top": "permitted", "agreement": 1.0, "input_tokens": 202}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30129610560834, "probs": [0.8174524855587436, 0.08365642576901129, 0.098891088672245], "top": "insufficient", "agreement": 1.0, "input_tokens": 204}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98417308554053, "probs": [0.017660819839882137, 0.04340570512192549, 0.9389334750381924], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.75598997436464, "probs": [0.06132539226815876, 0.8373735739130129, 0.10130103381882835], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.35578685812652, "probs": [0.887606329608409, 0.06778068214997746, 0.04461298824161357], "top": "prohibited", "agreement": 1.0, "input_tokens": 213}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.96964189223945, "probs": [0.6913075067568434, 0.07074700515018907, 0.23794548809296753], "top": "insufficient", "agreement": 1.0, "input_tokens": 212}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25284614413977, "probs": [0.9055821252398959, 0.059736224410951576, 0.03468165034915258], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22691602818668, "probs": [0.034796330955504114, 0.9473839012637663, 0.017819767780729617], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3523279260844, "probs": [0.06676731835885638, 0.04041944853804973, 0.8928132331030938], "top": "permitted", "agreement": 1.0, "input_tokens": 208}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42652802541852, "probs": [0.0792938381036179, 0.09571489465194817, 0.8249912672444338], "top": "insufficient", "agreement": 1.0, "input_tokens": 215}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1836758852005, "probs": [0.08689479609139293, 0.6606481239315956, 0.2524570799770115], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.70327991060913, "probs": [0.8171805101277473, 0.07074519344147624, 0.11207429643077639], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12545495294034, "probs": [0.103178626971303, 0.7367489124270009, 0.16007246060169616], "top": "prohibited", "agreement": 1.0, "input_tokens": 209}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16906506381929, "probs": [0.08560085550345033, 0.3403340342335013, 0.5740651102630484], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.15019483864307, "probs": [0.017814439341391167, 0.9671155236327101, 0.015070037025898734], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.20701506175101, "probs": [0.9513649777837366, 0.01716163808274369, 0.03147338413351958], "top": "B", "agreement": 1.0, "input_tokens": 201}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.73341098986566, "probs": [0.025903351803450637, 0.9255875837884197, 0.048509064408129646], "top": "B", "agreement": 1.0, "input_tokens": 192}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98396283946931, "probs": [0.019189428631759722, 0.9189139403005832, 0.061896631067657096], "top": "A", "agreement": 1.0, "input_tokens": 192}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31381703913212, "probs": [0.016185500037668298, 0.9553471934068705, 0.028467306555461265], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06533412262797, "probs": [0.014798671264711157, 0.9697705732073082, 0.015430755527980513], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3068870678544, "probs": [0.025560818133635414, 0.022080003391839246, 0.9523591784745253], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19623500294983, "probs": [0.05311118840492001, 0.6811115463281155, 0.2657772652669644], "top": "A", "agreement": 1.0, "input_tokens": 190}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.34080700017512, "probs": [0.026144179410345973, 0.9539350561208116, 0.019920764468842493], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.7771000303328, "probs": [0.9592258084083156, 0.024179546083237823, 0.016594645508446512], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.7072221301496, "probs": [0.015353470774838725, 0.9649135180343748, 0.019733011190786628], "top": "B", "agreement": 1.0, "input_tokens": 197}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.54218811355531, "probs": [0.6229250573303873, 0.29341037282098503, 0.08366456984862763], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16598518006504, "probs": [0.9523442934756741, 0.028977529107373938, 0.018678177416952112], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.28223705478013, "probs": [0.9509486773226812, 0.026062332698517814, 0.022988989978801007], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.676459915936, "probs": [0.028605829867980576, 0.9599959678670565, 0.01139820226496276], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.36008700728416, "probs": [0.027524206942156276, 0.10494883289705875, 0.8675269601607849], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27655598707497, "probs": [0.025763001405599038, 0.9598922160922182, 0.014344782502182778], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.81528099626303, "probs": [0.01775221293583573, 0.018510449440082843, 0.9637373376240814], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07745388895273, "probs": [0.05805098351023624, 0.04331699912016453, 0.8986320173695993], "top": "B", "agreement": 1.0, "input_tokens": 200}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.23310606367886, "probs": [0.40713917224854596, 0.04723599385140831, 0.5456248339000457], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.65796897187829, "probs": [0.9628824551152317, 0.020107611199832023, 0.017009933684936158], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.94616208598018, "probs": [0.017791047056264133, 0.965845589126288, 0.016363363817447876], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31268687918782, "probs": [0.9697536223339248, 0.01200584747578968, 0.018240530190285528], "top": "A", "agreement": 1.0, "input_tokens": 194}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.65730889700353, "probs": [0.46166613268095874, 0.4246186674610029, 0.11371519985803841], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 192}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.74538989365101, "probs": [0.9434733126902809, 0.03465269924248234, 0.021873988067236654], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.02151306718588, "probs": [0.05653147014528852, 0.049865123655206005, 0.8936034061995054], "top": "supported", "agreement": 1.0, "input_tokens": 214}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39991687983274, "probs": [0.9646005420197569, 0.019726610085808626, 0.01567284789443457], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.55080890096724, "probs": [0.011200303179937986, 0.9433281975088483, 0.045471499311213706], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.66023907065392, "probs": [0.04435070832583449, 0.8823880770841831, 0.07326121458998254], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.23385694250464, "probs": [0.38436219601806726, 0.39248484865749256, 0.22315295532444018], "top": "prohibited", "agreement": 0.3333333333333333, "input_tokens": 229}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.0447840243578, "probs": [0.038671196720481874, 0.051824945670009545, 0.9095038576095087], "top": "permitted", "agreement": 1.0, "input_tokens": 222}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.33729800954461, "probs": [0.9386225473366789, 0.024670771221418437, 0.03670668144190261], "top": "insufficient", "agreement": 1.0, "input_tokens": 221}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32930702902377, "probs": [0.06338502047709155, 0.20445210704639485, 0.7321628724765136], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.87917205318809, "probs": [0.01023906219590939, 0.9776572412337166, 0.012103696570374124], "top": "A", "agreement": 1.0, "input_tokens": 210}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.62321899831295, "probs": [0.08343895995982653, 0.048442981731284813, 0.8681180583088886], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09425401687622, "probs": [0.028590681320234453, 0.07026846391461965, 0.9011408547651459], "top": "insufficient", "agreement": 1.0, "input_tokens": 211}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.82056089676917, "probs": [0.9485370017696566, 0.021991387483579136, 0.02947161074676416], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.44618801213801, "probs": [0.1476984273576982, 0.06533845399896182, 0.78696311864334], "top": "supported", "agreement": 1.0, "input_tokens": 227}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9405219964683, "probs": [0.9659641119707486, 0.01300233393992522, 0.021033554089326202], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.44208786450326, "probs": [0.9452042665175415, 0.037745228086007734, 0.01705050539645064], "top": "insufficient", "agreement": 1.0, "input_tokens": 202}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.7328298445791, "probs": [0.7928775526060625, 0.1207272484983066, 0.08639519889563085], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.90913293324411, "probs": [0.4902958520500278, 0.23093932363732236, 0.2787648243126498], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 231}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13704502955079, "probs": [0.08132590401657058, 0.05465960600149562, 0.8640144899819338], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.0677141174674, "probs": [0.2290171921944468, 0.6937872957432885, 0.07719551206226469], "top": "insufficient", "agreement": 1.0, "input_tokens": 223}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.88581284880638, "probs": [0.030141465776599607, 0.9500195362761745, 0.019838997947225906], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.10445386171341, "probs": [0.025305433525381633, 0.03185065213808263, 0.9428439143365358], "top": "B", "agreement": 1.0, "input_tokens": 222}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.90737203508615, "probs": [0.05059101952456338, 0.04279722164027021, 0.9066117588351665], "top": "B", "agreement": 1.0, "input_tokens": 215}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07104406133294, "probs": [0.012768350746295443, 0.01860437029559649, 0.9686272789581081], "top": "insufficient", "agreement": 1.0, "input_tokens": 201}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.02462415024638, "probs": [0.9208322432392101, 0.04082517423414474, 0.038342582526645214], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19009595736861, "probs": [0.8434852492825246, 0.05011598360172626, 0.10639876711574907], "top": "supported", "agreement": 1.0, "input_tokens": 219}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.15409498475492, "probs": [0.023619578875477214, 0.01956735197412583, 0.9568130691503969], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.04633304663002, "probs": [0.8650697558641954, 0.10248576421341996, 0.03244447992238454], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.02380412071943, "probs": [0.10549476831325036, 0.0754944842886127, 0.819010747398137], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.61358912289143, "probs": [0.335350091795104, 0.11069809870287566, 0.5539518095020204], "top": "prohibited", "agreement": 1.0, "input_tokens": 221}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14674498699605, "probs": [0.8825022468959398, 0.07026933060840494, 0.047228422495655314], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.93841208517551, "probs": [0.06932232571305925, 0.9077945737563922, 0.022883100530548473], "top": "insufficient", "agreement": 1.0, "input_tokens": 220}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.15473480336368, "probs": [0.009281733826954271, 0.9839356748468795, 0.006782591326166279], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12535413727164, "probs": [0.008340312669325892, 0.9815921692199244, 0.010067518110749797], "top": "B", "agreement": 1.0, "input_tokens": 230}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98450300656259, "probs": [0.012023447059675983, 0.9711751830662149, 0.016801369874109195], "top": "B", "agreement": 1.0, "input_tokens": 223}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.80464994907379, "probs": [0.011643930956106696, 0.9806920781093156, 0.007663990934577689], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16871511936188, "probs": [0.900478485395825, 0.023178368052708644, 0.07634314655146633], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.04756402224302, "probs": [0.06499692312355161, 0.047496249800495424, 0.887506827075953], "top": "supported", "agreement": 1.0, "input_tokens": 224}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.89901201799512, "probs": [0.3922836361905562, 0.48352898160684454, 0.12418738220259931], "top": "insufficient", "agreement": 0.3333333333333333, "input_tokens": 220}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.6339690219611, "probs": [0.0560499070635644, 0.020116182060737196, 0.9238339108756985], "top": "insufficient", "agreement": 1.0, "input_tokens": 208}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.95372412353754, "probs": [0.17500642472273137, 0.21124871005668264, 0.613744865220586], "top": "permitted", "agreement": 1.0, "input_tokens": 227}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84458087012172, "probs": [0.5914884424684399, 0.035143500631624254, 0.37336805689993585], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 228}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.86592212691903, "probs": [0.9046569284481161, 0.042704941598692904, 0.05263812995319102], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41308694519103, "probs": [0.029736313438628644, 0.033014009977779125, 0.9372496765835922], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.47394684888422, "probs": [0.7432028429041843, 0.03582480375679601, 0.2209723533390198], "top": "A", "agreement": 1.0, "input_tokens": 221}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92841200903058, "probs": [0.06632110060786917, 0.8329169964218126, 0.10076190297031828], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.8825029283762, "probs": [0.7397871533568318, 0.1860713523015022, 0.07414149434166593], "top": "A", "agreement": 1.0, "input_tokens": 207}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.76130107417703, "probs": [0.9248773731002589, 0.014113602375072627, 0.06100902452466855], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.93099200539291, "probs": [0.9556486965100951, 0.021248690458660385, 0.02310261303124464], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.75966101512313, "probs": [0.010853780626859795, 0.015814708468139885, 0.9733315109050004], "top": "supported", "agreement": 1.0, "input_tokens": 212}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.78304110094905, "probs": [0.9606317563420671, 0.018450800471885003, 0.020917443186047876], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.90428190678358, "probs": [0.8027819128710822, 0.011505476315571077, 0.18571261081334672], "top": "contradicted", "agreement": 0.6666666666666666, "input_tokens": 203}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.71060988493264, "probs": [0.8487397319059693, 0.05482797840153974, 0.09643228969249089], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.79018115438521, "probs": [0.019700876806489794, 0.943405397557902, 0.036893725635608235], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.20129511132836, "probs": [0.9481975719388965, 0.021083015776241384, 0.030719412284862045], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06828408688307, "probs": [0.11594495476456004, 0.2790651305948641, 0.6049899146405758], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07413395680487, "probs": [0.011620895003629282, 0.009627189199890869, 0.9787519157964798], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.68773986212909, "probs": [0.9712341256474011, 0.015134214927072205, 0.013631659425526738], "top": "B", "agreement": 1.0, "input_tokens": 212}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.55650789663196, "probs": [0.9748621568921099, 0.014266993806726613, 0.010870849301163494], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.4433368369937, "probs": [0.01218702517662872, 0.9640155969735945, 0.02379737784977667], "top": "insufficient", "agreement": 1.0, "input_tokens": 191}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06905405782163, "probs": [0.05216159441960384, 0.25033305566222464, 0.6975053499181716], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11510400101542, "probs": [0.0744428867114499, 0.8598927356841719, 0.06566437760437807], "top": "supported", "agreement": 1.0, "input_tokens": 221}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4208381101489, "probs": [0.031673836031138235, 0.010902017412968387, 0.9574241465558934], "top": "supported", "agreement": 1.0, "input_tokens": 229}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17571517266333, "probs": [0.9598638711537073, 0.029206330829344665, 0.01092979801694802], "top": "insufficient", "agreement": 1.0, "input_tokens": 226}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.40474788285792, "probs": [0.3126088342455135, 0.527299092068065, 0.1600920736864215], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.87994202412665, "probs": [0.018596715235427216, 0.8540525288330012, 0.1273507559315715], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.45206791721284, "probs": [0.2682095319185024, 0.5694223884119577, 0.16236807966953995], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.52172788791358, "probs": [0.5197715159798874, 0.040530340274099394, 0.4396981437460133], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 224}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17407604493201, "probs": [0.03677949530401433, 0.9404844838908847, 0.02273602080510092], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.91013200953603, "probs": [0.09722552658518739, 0.7089119171844285, 0.19386255623038412], "top": "B", "agreement": 1.0, "input_tokens": 218}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.70784991048276, "probs": [0.0446355264117619, 0.029378934066784906, 0.9259855395214532], "top": "B", "agreement": 1.0, "input_tokens": 209}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.70177978277206, "probs": [0.8956127457317613, 0.023540301062398936, 0.08084695320583983], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22924596443772, "probs": [0.9215420659013949, 0.05709241087032677, 0.021365523228278453], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.94459210895002, "probs": [0.024650674719169527, 0.05690076079826144, 0.9184485644825692], "top": "contradicted", "agreement": 1.0, "input_tokens": 215}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30767706222832, "probs": [0.017881542110937673, 0.0314503320010902, 0.9506681258879721], "top": "contradicted", "agreement": 1.0, "input_tokens": 236}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.78564002551138, "probs": [0.015971281239954956, 0.021403804942628207, 0.9626249138174169], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07165500894189, "probs": [0.9469841068616602, 0.027062216114961006, 0.025953677023378714], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.04044406116009, "probs": [0.9655262695437641, 0.017417092730978554, 0.01705663772525738], "top": "supported", "agreement": 1.0, "input_tokens": 241}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.97531294822693, "probs": [0.9523229876734454, 0.019887100792948465, 0.02778991153360623], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.55496702343225, "probs": [0.02944395390697739, 0.022909170093210977, 0.9476468759998116], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.69904006458819, "probs": [0.028467305657075486, 0.01618550022217239, 0.9553471941207522], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92244199849665, "probs": [0.9626180927581529, 0.027508993151357353, 0.009872914090489732], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.61895912885666, "probs": [0.01216177373585067, 0.025820060282101747, 0.9620181659820476], "top": "supported", "agreement": 1.0, "input_tokens": 216}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.76713092066348, "probs": [0.008947931903485177, 0.02245643770922697, 0.9685956303872878], "top": "supported", "agreement": 1.0, "input_tokens": 237}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.7069001942873, "probs": [0.03490400868942244, 0.07256938070358598, 0.8925266106069916], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.20225507207215, "probs": [0.8560321676779682, 0.10355825471147521, 0.04040957761055657], "top": "contradicted", "agreement": 1.0, "input_tokens": 209}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12030404061079, "probs": [0.9275028632254695, 0.04760344873379892, 0.024893688040731596], "top": "contradicted", "agreement": 1.0, "input_tokens": 230}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09906406514347, "probs": [0.9261949727483907, 0.021028996331540862, 0.0527760309200684], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.26782716624439, "probs": [0.03674381458625419, 0.02368412668428037, 0.9395720587294654], "top": "supported", "agreement": 1.0, "input_tokens": 222}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.90970290265977, "probs": [0.026670715956627202, 0.020321964093249996, 0.9530073199501229], "top": "supported", "agreement": 1.0, "input_tokens": 243}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.93341321125627, "probs": [0.08330838841015804, 0.7031959025468014, 0.21349570904304058], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.10100517421961, "probs": [0.18172827933132565, 0.6645395854888777, 0.15373213517979664], "top": "prohibited", "agreement": 1.0, "input_tokens": 223}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9823029898107, "probs": [0.08895150060315864, 0.816337598890086, 0.09471090050675535], "top": "prohibited", "agreement": 1.0, "input_tokens": 244}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.05649400874972, "probs": [0.0293692258945804, 0.11923451516955454, 0.851396258935865], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1215750630945, "probs": [0.7559638022373625, 0.22951644325448917, 0.014519754508148302], "top": "permitted", "agreement": 1.0, "input_tokens": 220}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14070489443839, "probs": [0.9180643670041458, 0.055699863652112935, 0.026235769343741142], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.65426882728934, "probs": [0.8545877320672847, 0.07877388866940015, 0.06663837926331509], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01189390197396, "probs": [0.10813821274832802, 0.06970313933726799, 0.822158647914404], "top": "prohibited", "agreement": 1.0, "input_tokens": 219}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09787383489311, "probs": [0.04460783183966404, 0.029981179056752127, 0.9254109891035839], "top": "prohibited", "agreement": 1.0, "input_tokens": 240}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.91549200378358, "probs": [0.9383472668786358, 0.03989758917731366, 0.021755143944050585], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.85541191883385, "probs": [0.019454336721104887, 0.06822606373845527, 0.9123195995404397], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.99751404859126, "probs": [0.015551699476841353, 0.0471124973010445, 0.9373358032221141], "top": "permitted", "agreement": 1.0, "input_tokens": 246}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21135502494872, "probs": [0.0377743579001357, 0.05503984914430352, 0.9071857929555608], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.82518211938441, "probs": [0.9057645436591656, 0.057300812015498664, 0.03693464432533565], "top": "prohibited", "agreement": 1.0, "input_tokens": 227}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.02380412071943, "probs": [0.9388430475426657, 0.037491202062820105, 0.023665750394514205], "top": "prohibited", "agreement": 1.0, "input_tokens": 248}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.51878793537617, "probs": [0.28824200219090185, 0.5868851532425197, 0.12487284456657842], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07629485800862, "probs": [0.5631497490596038, 0.19793031332732086, 0.2389199376130753], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.95753213763237, "probs": [0.07442810557243966, 0.7426469601625133, 0.18292493426504697], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.79146102443337, "probs": [0.021424613092898774, 0.963560742046841, 0.01501464486026032], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01755308359861, "probs": [0.01601169462902707, 0.9650607227828113, 0.01892758258816167], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08328396826982, "probs": [0.01772358905716697, 0.9621833982502599, 0.020093012692573263], "top": "A", "agreement": 1.0, "input_tokens": 232}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.63964892551303, "probs": [0.038484166247206124, 0.9437641516667067, 0.017751682086087198], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13378493487835, "probs": [0.01638885815779191, 0.9473306235405403, 0.036280518301667765], "top": "B", "agreement": 1.0, "input_tokens": 219}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21683609113097, "probs": [0.01830968230910091, 0.9532844759985375, 0.028405841692361555], "top": "B", "agreement": 1.0, "input_tokens": 240}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.26257706806064, "probs": [0.023557953254122476, 0.9543166602232142, 0.022125386522663444], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12111405842006, "probs": [0.031413960782038314, 0.9495687268473656, 0.01901731237059622], "top": "A", "agreement": 1.0, "input_tokens": 214}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13328504748642, "probs": [0.07602225290100428, 0.8966935357050799, 0.027284211393915826], "top": "A", "agreement": 1.0, "input_tokens": 235}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31315696425736, "probs": [0.023803230827026532, 0.0318997393139744, 0.944297029858999], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92733307182789, "probs": [0.9537738809196197, 0.029021027054749942, 0.017205092025630454], "top": "B", "agreement": 1.0, "input_tokens": 214}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.28281407803297, "probs": [0.964600540916227, 0.019726610712202018, 0.015672848371570857], "top": "B", "agreement": 1.0, "input_tokens": 235}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.76316115818918, "probs": [0.01331158113941839, 0.9684719956116256, 0.018216423248955952], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92315189912915, "probs": [0.03555637420973125, 0.9480424533538477, 0.016401172436420974], "top": "A", "agreement": 1.0, "input_tokens": 219}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.52477999590337, "probs": [0.02192392382604651, 0.9656109109209706, 0.012465165252982892], "top": "A", "agreement": 1.0, "input_tokens": 240}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17497500404716, "probs": [0.019595497666027976, 0.022215172638678914, 0.9581893296952932], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16752605326474, "probs": [0.9596984415322811, 0.02225016024721796, 0.01805139822050088], "top": "B", "agreement": 1.0, "input_tokens": 217}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.82863103970885, "probs": [0.9679216672147071, 0.013872260676720507, 0.01820607210857243], "top": "B", "agreement": 1.0, "input_tokens": 238}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06158415228128, "probs": [0.02350526733620643, 0.04401818593722302, 0.9324765467265705], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.8507911618799, "probs": [0.9307059696632914, 0.042134929049560656, 0.027159101287147974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17631587572396, "probs": [0.9518944397002193, 0.030200951503065683, 0.01790460879671502], "top": "contradicted", "agreement": 1.0, "input_tokens": 248}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.86692213453352, "probs": [0.0734946189438627, 0.8668797160257922, 0.05962566503034527], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84097106382251, "probs": [0.06641491931728377, 0.8517220237244112, 0.08186305695830509], "top": "prohibited", "agreement": 1.0, "input_tokens": 239}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.88027304038405, "probs": [0.026075603104276424, 0.9317425408247122, 0.04218185607101137], "top": "prohibited", "agreement": 1.0, "input_tokens": 260}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13479495421052, "probs": [0.8272206536224163, 0.11345127657827278, 0.05932806979931101], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30668590217829, "probs": [0.10398356939461242, 0.36466878809364145, 0.5313476425117462], "top": "A", "agreement": 0.3333333333333333, "input_tokens": 227}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19572580233216, "probs": [0.047442131153065406, 0.31816121638654793, 0.6343966524603867], "top": "A", "agreement": 1.0, "input_tokens": 248}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06471409462392, "probs": [0.026110501601006305, 0.021183264898242012, 0.9527062335007518], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.89221196621656, "probs": [0.9405169120595193, 0.024208959715331872, 0.035274128225148874], "top": "contradicted", "agreement": 1.0, "input_tokens": 237}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93056306242943, "probs": [0.9538802773453129, 0.023059861528293313, 0.023059861126393875], "top": "contradicted", "agreement": 1.0, "input_tokens": 258}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.33315688371658, "probs": [0.11358428062013931, 0.09215012355248668, 0.7942655958273741], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19191599451005, "probs": [0.6104725669434667, 0.2700592902850353, 0.11946814277149793], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 244}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.92946386896074, "probs": [0.8448473542913075, 0.08645996139090424, 0.06869268431778815], "top": "prohibited", "agreement": 1.0, "input_tokens": 265}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.97641214169562, "probs": [0.0265024583113502, 0.9469950831682415, 0.02650245852040832], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.48200900666416, "probs": [0.04213492809402675, 0.9307059697792428, 0.027159102126730544], "top": "A", "agreement": 1.0, "input_tokens": 233}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14860413968563, "probs": [0.02140380544189133, 0.962624913264518, 0.01597128129359056], "top": "A", "agreement": 1.0, "input_tokens": 254}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38465809077024, "probs": [0.04515777084298974, 0.03741040508103143, 0.9174318240759788], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.95074279606342, "probs": [0.9100932546372905, 0.041201749446211355, 0.048704995916498045], "top": "contradicted", "agreement": 1.0, "input_tokens": 234}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.78034189529717, "probs": [0.9366656772497187, 0.038194709842211357, 0.025139612908070024], "top": "contradicted", "agreement": 1.0, "input_tokens": 255}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31453718431294, "probs": [0.868308675840529, 0.07676005878826689, 0.0549312653712041], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11085484176874, "probs": [0.15789450752928103, 0.11538080349900755, 0.7267246889717114], "top": "prohibited", "agreement": 1.0, "input_tokens": 235}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.83383107930422, "probs": [0.08083885614456791, 0.06032105358305889, 0.8588400902723732], "top": "prohibited", "agreement": 1.0, "input_tokens": 256}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06573412567377, "probs": [0.009236476474052145, 0.9791380440055677, 0.011625479520380164], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.93744187988341, "probs": [0.014594034131288612, 0.9765710625943955, 0.008834903274315823], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.96883506514132, "probs": [0.0070908823458139675, 0.986522232315347, 0.006386885338839195], "top": "A", "agreement": 1.0, "input_tokens": 259}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14887515455484, "probs": [0.07423043782078662, 0.012813700792315847, 0.9129558613868974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09843402542174, "probs": [0.9080827305344887, 0.025413463174314043, 0.06650380629119718], "top": "contradicted", "agreement": 1.0, "input_tokens": 238}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.20908695273101, "probs": [0.912724042631006, 0.024497031790175153, 0.06277892557881881], "top": "contradicted", "agreement": 1.0, "input_tokens": 259}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.1586238425225, "probs": [0.44029144656423663, 0.4049592579435585, 0.15474929549220479], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 227}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14268395490944, "probs": [0.36520001887064335, 0.21202778871080402, 0.4227721924185527], "top": "permitted", "agreement": 0.3333333333333333, "input_tokens": 238}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.09033610671759, "probs": [0.0870101345530018, 0.5040542344698868, 0.4089356309771113], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 259}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.8780910205096, "probs": [0.4770731173522989, 0.04585376828987368, 0.4770731143578275], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 221}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.83801085501909, "probs": [0.39895641057043185, 0.0317669442524896, 0.5692766451770785], "top": "B", "agreement": 1.0, "input_tokens": 232}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13637494295835, "probs": [0.3779951490486449, 0.03557902847506642, 0.5864258224762886], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 253}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.73026102408767, "probs": [0.02269220266989126, 0.01879907857817761, 0.9585087187519311], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.26010487228632, "probs": [0.9501164329419498, 0.021572126471092922, 0.028311440586957334], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92062289267778, "probs": [0.9561058300755273, 0.022635315404623546, 0.021258854519849178], "top": "contradicted", "agreement": 1.0, "input_tokens": 245}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.7083989251405, "probs": [0.09448034891197682, 0.0384419614966505, 0.8670776895913728], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08244391530752, "probs": [0.7989049560014004, 0.057297274549302536, 0.143797769449297], "top": "prohibited", "agreement": 1.0, "input_tokens": 229}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.87414221279323, "probs": [0.9033714574378554, 0.03317980421079573, 0.06344873835134875], "top": "prohibited", "agreement": 1.0, "input_tokens": 250}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.7984310425818, "probs": [0.9807799827157122, 0.008509520309325268, 0.010710496974962463], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.61058018356562, "probs": [0.009842123709939055, 0.01026250243418525, 0.9798953738558757], "top": "A", "agreement": 1.0, "input_tokens": 217}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.2257651463151, "probs": [0.011627702527445261, 0.00904705287357599, 0.9793252445989787], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08419410325587, "probs": [0.6618883595686608, 0.28081044656836285, 0.05730119386297648], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.91706198081374, "probs": [0.06488320495606428, 0.37587178359700457, 0.5592450114469311], "top": "contradicted", "agreement": 1.0, "input_tokens": 235}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38519686087966, "probs": [0.06100397866886853, 0.1887116244179442, 0.7502843969131873], "top": "contradicted", "agreement": 1.0, "input_tokens": 256}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22031621076167, "probs": [0.17125015092489698, 0.5080506282282206, 0.32069922084688257], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.42017889767885, "probs": [0.1514529519738628, 0.627869626193126, 0.22067742183301117], "top": "permitted", "agreement": 1.0, "input_tokens": 237}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.79001291282475, "probs": [0.17013095481276255, 0.7202069629772057, 0.10966208221003176], "top": "permitted", "agreement": 1.0, "input_tokens": 258}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92878197692335, "probs": [0.023818955965634256, 0.9449208465220887, 0.031260197512277065], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9507730640471, "probs": [0.04244531776287583, 0.9375620734741275, 0.01999260876299673], "top": "A", "agreement": 1.0, "input_tokens": 224}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1125640515238, "probs": [0.028930382730149296, 0.9507948329241313, 0.02027478434571944], "top": "A", "agreement": 1.0, "input_tokens": 245}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.2224799208343, "probs": [0.227340819830238, 0.7726591801697621], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.54156290926039, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.36634085327387, "probs": [0.16800751383797227, 0.8319924861620277], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.97272691316903, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22090901248157, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27856113947928, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.24301907606423, "probs": [0.20097170058635486, 0.7990282994136452], "top": "ordinary", "agreement": 1.0, "input_tokens": 254}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.98715589381754, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.2423890363425, "probs": [0.13950622912843208, 0.8604937708715679], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47289300337434, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.29039010033011, "probs": [0.3067002008775075, 0.6932997991224925], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25836005434394, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.15584914572537, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "agreement": 1.0, "input_tokens": 256}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63464394398034, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.29663089290261, "probs": [0.07740461434958126, 0.9225953856504188], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.880935061723, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.37011107988656, "probs": [0.09196032674706488, 0.9080396732529351], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.09374809078872, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.59197400324047, "probs": [0.1552608265624769, 0.844739173437523], "top": "ordinary", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.12798793241382, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.20007881894708, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.02194800972939, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0895680822432, "probs": [0.32019975839988934, 0.6798002416001107], "top": "ordinary", "agreement": 1.0, "input_tokens": 255}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.7541941665113, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.40705198980868, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.23521901667118, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.50419196113944, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.42402208410203, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.44741201773286, "probs": [0.13578298208575287, 0.8642170179142472], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.44605088979006, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03182787634432, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.78236411325634, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.60102389752865, "probs": [0.9710643563300382, 0.02893564366996184], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.46341213956475, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30208006128669, "probs": [0.9673224283590844, 0.032677571640915566], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.21616904623806, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.51858299039304, "probs": [0.9350120677910418, 0.06498793220895807], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.23841997236013, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84819597750902, "probs": [0.8405803537312423, 0.1594196462687578], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.35239988565445, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.6207642108202, "probs": [0.9290542127600319, 0.07094578723996812], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38429186306894, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.93546695634723, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.28442102111876, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.00706796720624, "probs": [0.9516645319845058, 0.048335468015494064], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66723487898707, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90192700363696, "probs": [0.9439168350492196, 0.056083164950780265], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0566688105464, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.21789897792041, "probs": [0.905386513027929, 0.09461348697207088], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18360907770693, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.35832193307579, "probs": [0.9156019647894573, 0.0843980352105427], "top": "react", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.7594440318644, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.16857916116714, "probs": [0.6155100736384166, 0.38448992636158347], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18823798373342, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.59790396690369, "probs": [0.6932998086882517, 0.30670019131174836], "top": "react", "agreement": 1.0, "input_tokens": 268}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.88707503862679, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62644481286407, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.93356596305966, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4756220113486, "probs": [0.915601962671152, 0.08439803732884785], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.97568712197244, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.59467390738428, "probs": [0.40698895046193323, 0.5930110495380667], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.42366212792695, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.42155198380351, "probs": [0.36247441705939704, 0.637525582940603], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.85423590615392, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.3101010080427, "probs": [0.908039679856927, 0.09196032014307312], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.6581030972302, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.95610785856843, "probs": [0.3481052260909569, 0.6518947739090432], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.86958503350616, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.03076693601906, "probs": [0.5235093404303358, 0.4764906595696642], "top": "react", "agreement": 0.5, "input_tokens": 255}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48832199163735, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22572883963585, "probs": [0.2743951410804348, 0.7256048589195652], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.87317504920065, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.48452110029757, "probs": [0.2112354509623397, 0.7887645490376604], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11741811968386, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.99809707142413, "probs": [0.12511350715617336, 0.8748864928438266], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.66676300019026, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.61183196492493, "probs": [0.27439514286499256, 0.7256048571350073], "top": "ordinary", "agreement": 1.0, "input_tokens": 181}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.82477491348982, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.7731950096786, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.68515289574862, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.73482405394316, "probs": [0.4222145135481339, 0.5777854864518661], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.6708231009543, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.88368595577776, "probs": [0.1472092305510008, 0.8527907694489993], "top": "ordinary", "agreement": 1.0, "input_tokens": 183}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.02580787613988, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.08476688154042, "probs": [0.11201237833636817, 0.8879876216636319], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.93775598332286, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.70550415664911, "probs": [0.12172004831180805, 0.878279951688192], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.88940497487783, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.08671893365681, "probs": [0.17243807104776526, 0.8275619289522348], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.88573509827256, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.94388687983155, "probs": [0.16366828025224447, 0.8363317197477556], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.12652901560068, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.59551193378866, "probs": [0.3000708042310285, 0.6999291957689716], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.87468612007797, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.91828505508602, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.42248989455402, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.54867199622095, "probs": [0.10592235643160026, 0.8940776435683997], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.9121460095048, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.20382385700941, "probs": [0.19108583142179145, 0.8089141685782085], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.24842004850507, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.90460509061813, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0779179353267, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.87610592134297, "probs": [0.7614507841997993, 0.23854921580020058], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.76137396506965, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.905715925619, "probs": [0.7887645476620578, 0.2112354523379421], "top": "react", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.84253500215709, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.53605187684298, "probs": [0.4921584130878123, 0.5078415869121877], "top": "ordinary", "agreement": 0.5, "input_tokens": 191}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.81449497491121, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.78241393901408, "probs": [0.293524032474578, 0.7064759675254221], "top": "ordinary", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7478059232235, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.64276305027306, "probs": [0.5156793119007109, 0.484320688099289], "top": "react", "agreement": 0.5, "input_tokens": 186}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.74044412001967, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.82638517022133, "probs": [0.6302458313412554, 0.3697541686587445], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.63580304384232, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.07793795876205, "probs": [0.7318060925310856, 0.2681939074689144], "top": "react", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.54339186288416, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.64040191471577, "probs": [0.5469149637193962, 0.45308503628060387], "top": "react", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.40888110734522, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.86490490473807, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.34890982322395, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.77834499068558, "probs": [0.44532390802961846, 0.5546760919703815], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.82116394303739, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.68361295387149, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.74451399967074, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0575480479747, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "agreement": 1.0, "input_tokens": 195}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.9630970377475, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.80591400153935, "probs": [0.2501313189948441, 0.749868681005156], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.54497208446264, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.99332613870502, "probs": [0.17696061199867663, 0.8230393880013234], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.60158206149936, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.50628098659217, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.7149428781122, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.65423298627138, "probs": [0.2870615213045163, 0.7129384786954838], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.83179608546197, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8271171208471, "probs": [0.4145813330827104, 0.5854186669172896], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.63504192046821, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.60715300031006, "probs": [0.44532391331269805, 0.554676086687302], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.87368611246347, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.66658209078014, "probs": [0.35525721497515955, 0.6447427850248404], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.76243381015956, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28345097973943, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.21357903815806, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.84069409035146, "probs": [0.8269315768853935, 0.12075476980815658, 0.05231365330644987], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.52226805500686, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.98068709112704, "probs": [0.9608203282304785, 0.03917967176952151], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0840979591012, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.17312805354595, "probs": [0.9620325019409397, 0.02528607877990563, 0.012681419279154706], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.50500894896686, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.44153204374015, "probs": [0.06890580795431694, 0.931094192045683], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.49001304060221, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.25346906110644, "probs": [0.8320645593484999, 0.04643111322039759, 0.12150432743110248], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.51534895785153, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.37559300847352, "probs": [0.9544700034284429, 0.04552999657155723], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.37763213925064, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.2285680565983, "probs": [0.9527150999124248, 0.019483546419057494, 0.027801353668517746], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76395085081458, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1714401151985, "probs": [0.12172004535178665, 0.8782799546482134], "top": "holds", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.98170695826411, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.89989500679076, "probs": [0.0645643144235774, 0.035205281163747985, 0.9002304044126745], "top": "tavern", "agreement": 1.0, "input_tokens": 400}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.88727199845016, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99196796864271, "probs": [0.1862839992992625, 0.8137160007007376], "top": "elsewhere", "agreement": 1.0, "input_tokens": 360}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.98685600794852, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.75791301764548, "probs": [0.36775256751831553, 0.024771247006193476, 0.6074761854754909], "top": "tavern", "agreement": 0.6666666666666666, "input_tokens": 412}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.68512114696205, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02554913237691, "probs": [0.0796746792375593, 0.9203253207624406], "top": "holds", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.71940298192203, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.32526006177068, "probs": [0.08290919036094808, 0.8994500391535621, 0.017640770485489753], "top": "meet_wren", "agreement": 1.0, "input_tokens": 403}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.76875095628202, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.88169304654002, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 363}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.3444510474801, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.63272317126393, "probs": [0.6296525959845715, 0.35799812782967866, 0.012349276185749847], "top": "stay", "agreement": 0.6666666666666666, "input_tokens": 415}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60039104893804, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.45463295839727, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 351}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.86495612747967, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.73560318537056, "probs": [0.5514055185500297, 0.41145265329172886, 0.03714182815824141], "top": "stay", "agreement": 1.0, "input_tokens": 404}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.60888897068799, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.10339000448585, "probs": [0.9673224283553485, 0.03267757164465143], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.301459800452, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.90577398054302, "probs": [0.9603331749383929, 0.028023658411333054, 0.011643166650274083], "top": "stay", "agreement": 1.0, "input_tokens": 416}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65736005082726, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31400208547711, "probs": [0.7439389594906934, 0.2560610405093065], "top": "moved", "agreement": 1.0, "input_tokens": 345}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.91743499040604, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.9239650387317, "probs": [0.329927824572513, 0.5565122407492623, 0.11355993467822469], "top": "meet_wren", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.27944008633494, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.2778231613338, "probs": [0.02983020500731863, 0.9701697949926814], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.06048706918955, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.95724608376622, "probs": [0.7871268373024943, 0.18209101999191218, 0.030782142705593513], "top": "stay", "agreement": 1.0, "input_tokens": 410}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5607299786061, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97552709840238, "probs": [0.04979902878533396, 0.950200971214666], "top": "holds", "agreement": 1.0, "input_tokens": 334}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.8645861595869, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.0222060661763, "probs": [0.9109478037576992, 0.057628718316645906, 0.03142347792565483], "top": "stay", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.6227088663727, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.93075679242611, "probs": [0.9719328658769364, 0.028067134123063546], "top": "same", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.85462613031268, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.06281708553433, "probs": [0.9710720272034978, 0.01715460731299303, 0.01177336548350914], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.45039898715913, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.33768212608993, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.06268708594143, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.89108399674296, "probs": [0.1337836073050901, 0.8426345520770127, 0.0235818406178973], "top": "meet_wren", "agreement": 1.0, "input_tokens": 407}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.04667391069233, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.24207115359604, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.82436489872634, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.08536696620286, "probs": [0.7852826544019835, 0.20168841824349107, 0.013028927354525465], "top": "stay", "agreement": 1.0, "input_tokens": 419}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90627300553024, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76768616773188, "probs": [0.05776707671018535, 0.9422329232898147], "top": "holds", "agreement": 1.0, "input_tokens": 336}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.85357489995658, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.01028801500797, "probs": [0.706475960089318, 0.29352403991068193], "top": "stay", "agreement": 1.0, "input_tokens": 370}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.61342196539044, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.92020700313151, "probs": [0.9631149772558724, 0.036885022744127596], "top": "same", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.39826216734946, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.15122901834548, "probs": [0.9583890947004433, 0.04161090529955677], "top": "stay", "agreement": 1.0, "input_tokens": 381}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.35000103153288, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.08279892802238, "probs": [0.05608316364845092, 0.9439168363515491], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.10776822082698, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.29132205247879, "probs": [0.5156793134278208, 0.4843206865721793], "top": "stay", "agreement": 0.5, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.84378483705223, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.93139707669616, "probs": [0.9471470205154783, 0.05285297948452184], "top": "same", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.5252718180418, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.29654188267887, "probs": [0.9471470206637833, 0.05285297933621657], "top": "stay", "agreement": 1.0, "input_tokens": 389}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.71433309465647, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80591602809727, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 340}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.04817708022892, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.41890519112349, "probs": [0.7614507825686178, 0.23854921743138224], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.12846982292831, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0499090384692, "probs": [0.9583890946533792, 0.04161090534662085], "top": "same", "agreement": 1.0, "input_tokens": 353}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0696280002594, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.82446813769639, "probs": [0.9516645320340041, 0.04833546796599583], "top": "stay", "agreement": 1.0, "input_tokens": 385}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.00679608993232, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.96793798357248, "probs": [0.4921584129984156, 0.5078415870015844], "top": "holds", "agreement": 1.0, "input_tokens": 333}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.79930393956602, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.33216287568212, "probs": [0.07094578575062087, 0.9290542142493791], "top": "departure", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8478059861809, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86641707085073, "probs": [0.0820053924809899, 0.9179946075190101], "top": "elsewhere", "agreement": 1.0, "input_tokens": 346}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.01310696452856, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.71497493982315, "probs": [0.1724380702954841, 0.8275619297045159], "top": "departure", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.14415802061558, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.75304600968957, "probs": [0.26819391433993306, 0.7318060856600669], "top": "holds", "agreement": 1.0, "input_tokens": 339}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.00715697742999, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.13280000723898, "probs": [0.04833546845435023, 0.9516645315456497], "top": "departure", "agreement": 1.0, "input_tokens": 373}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.12870807759464, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28862098418176, "probs": [0.06310762735784174, 0.9368923726421583], "top": "elsewhere", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.76537399552763, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97866798378527, "probs": [0.17243806760921704, 0.8275619323907829], "top": "departure", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.764543954283, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.88521698117256, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 337}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.79462497495115, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.01142911799252, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "agreement": 1.0, "input_tokens": 371}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27025001682341, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.98304799385369, "probs": [0.9608203281857868, 0.039179671814213245], "top": "same", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.34787109680474, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.13951996527612, "probs": [0.9439168372028508, 0.05608316279714923], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.91307593509555, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.98136812075973, "probs": [0.07740461992444837, 0.9225953800755516], "top": "holds", "agreement": 1.0, "input_tokens": 343}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.93149586580694, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.22875114530325, "probs": [0.6589789156058212, 0.3410210843941787], "top": "stay", "agreement": 1.0, "input_tokens": 377}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.95862599089742, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.92899798974395, "probs": [0.9596222367905244, 0.040377763209475684], "top": "same", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.26541901379824, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.29016185738146, "probs": [0.9422329206296081, 0.05776707937039187], "top": "stay", "agreement": 1.0, "input_tokens": 388}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.77653403393924, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.07919890061021, "probs": [0.061278160236647276, 0.9387218397633527], "top": "holds", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.14783907309175, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.13600003160536, "probs": [0.8910699865839312, 0.10893001341606875], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.98532584495842, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04239792190492, "probs": [0.9663161960398492, 0.03368380396015082], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.56286209262908, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.17102986760437, "probs": [0.9673224283958624, 0.03267757160413767], "top": "stay", "agreement": 1.0, "input_tokens": 394}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.70887391455472, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8829759862274, "probs": [0.8527907664515249, 0.14720923354847515], "top": "moved", "agreement": 1.0, "input_tokens": 348}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.84509497508407, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02330813743174, "probs": [0.04161090472473705, 0.9583890952752631], "top": "outside", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.71203389205039, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.22551107406616, "probs": [0.05608316279714923, 0.9439168372028508], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.78399392776191, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.2404511179775, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "agreement": 1.0, "input_tokens": 390}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.81922399811447, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77765504457057, "probs": [0.07094578723996812, 0.9290542127600319], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.94576605595648, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9800191000104, "probs": [0.8040183777913431, 0.19598162220865686], "top": "stay", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.69935300387442, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04194902442396, "probs": [0.959622237525547, 0.040377762474453], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.89904602617025, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36850284971297, "probs": [0.9583890941641539, 0.041610905835846104], "top": "stay", "agreement": 1.0, "input_tokens": 396}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.7032831851393, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1971308812499, "probs": [0.7781221683337017, 0.22187783166629826], "top": "moved", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.88530506007373, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.26679101586342, "probs": [0.21123545145303588, 0.7887645485469642], "top": "outside", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.88695606216788, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19722005538642, "probs": [0.03688502246073314, 0.9631149775392668], "top": "elsewhere", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.51545195281506, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9868871383369, "probs": [0.755706186939545, 0.244293813060455], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.7644650246948, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.93279708735645, "probs": [0.12172004614068278, 0.8782799538593171], "top": "holds", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.17372890934348, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.27877108380198, "probs": [0.8879876230204682, 0.11201237697953184], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.99396688863635, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.97580100223422, "probs": [0.9558139330893122, 0.04418606691068789], "top": "same", "agreement": 1.0, "input_tokens": 366}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.1800920329988, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.20679298229516, "probs": [0.9631149782667298, 0.0368850217332701], "top": "stay", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.39887304604053, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.12142902426422, "probs": [0.09733498488397495, 0.9026650151160249], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.79385696165264, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.02063114568591, "probs": [0.8319924818822048, 0.16800751811779524], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.74070614948869, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.39185406453907, "probs": [0.9471470206637833, 0.05285297933621657], "top": "same", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85572711750865, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28940119966865, "probs": [0.955813931654481, 0.044186068345518914], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.02299691177905, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}], "seconds": 49.7, "failures": 0}, "repeat": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47056587599218, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 34.52}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.186692003160715, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.39}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949099972844124, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.24}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01212117448449, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.3}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96792991831899, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.23}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26231399923563, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.44}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83451912365854, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.09}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12949203327298, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.39}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08572196774185, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.25}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.214742973446846, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.48}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86621901579201, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 33.99}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88381891697645, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 33.97}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65766608901322, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 33.71}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.164621986448765, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.4}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.980181001126766, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 34.11}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19977312721312, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.38}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.058160873129964, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.33}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.215363001450896, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.45}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.964301019907, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.13}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38011512905359, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.6}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92786884494126, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.14}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14030189439654, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.38}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80759808048606, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.03}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.835988983511925, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.07}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94104915484786, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.08}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37241495400667, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.53}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92948003299534, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.19}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.99427317082882, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 35.89}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.639512008056045, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 37.63}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.502306047827005, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 34.66}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.054280983284116, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.25}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.003699922934175, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.21}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.153631914407015, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.32}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.882918793708086, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 33.93}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96640022099018, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.17}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42183488793671, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.6}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91818891093135, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.1}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09152201190591, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.31}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85842896811664, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.14}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39298484288156, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.65}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14849194511771, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.28}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06585103645921, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.3}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95539012551308, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.03}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.447514943778515, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.54}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.816759034991264, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 34.12}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27399301715195, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.33}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67322685942054, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 33.72}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15300210751593, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.41}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1353121008724, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.33}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14274289086461, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.2}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.862499080598354, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.13}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.894598975777626, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 33.99}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98674985207617, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.07}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03884105198085, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.22}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.086740834638476, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.35}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07441105507314, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.33}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.852288991212845, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 33.94}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92870889790356, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 33.95}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52058495208621, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 33.65}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29961393401027, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.42}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.739376839250326, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 33.79}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90882911905646, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 33.93}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.883380031213164, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 33.98}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.019910058006644, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 33.98}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.941710161045194, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.07}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08499111235142, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.01}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.421995144337416, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 33.5}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.442005006596446, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.35}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48383519612253, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 33.63}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03415999189019, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.25}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15664194896817, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.33}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1517320163548, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.36}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01534098945558, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.27}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40755491890013, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.45}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64271580055356, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 33.72}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03934093937278, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.25}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66908689774573, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 33.74}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7988300062716, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 35.61}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53889701329172, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.53}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18148195184767, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.31}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85363894328475, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.16}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26475406438112, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.47}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74852685071528, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 33.88}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1736919041723, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.32}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68577782809734, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 33.92}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12271200492978, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.33}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22999314777553, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.42}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24201302975416, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.49}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01649000868201, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.1}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27734414860606, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.41}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86705813743174, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.04}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.257533986121416, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 34.51}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.124202121049166, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.3}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.032641004770994, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.37}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18778281472623, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.44}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.309603998437524, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.46}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70626715756953, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 33.9}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15610294602811, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.45}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98396007530391, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.16}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19472209922969, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.4}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03726106323302, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.28}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23566303960979, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.42}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87102906405926, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.16}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.131591932848096, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.41}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.037011003121734, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.13}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95861995220184, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.02}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.707466933876276, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.03}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92787000909448, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.21}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.268602987751365, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.36}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10459196008742, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.29}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88245778903365, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.15}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16541198082268, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.3}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.661325953900814, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 33.75}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67119704186916, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.01}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.3757039885968, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 33.62}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1220619417727, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.32}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13174187578261, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.28}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21280302852392, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.37}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98331001214683, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.2}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52671601250768, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.67}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96029004640877, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.2}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.163072964176536, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.3}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.035831017419696, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.18}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.215433083474636, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.42}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70673700980842, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 33.87}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15965198352933, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.32}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71684698201716, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 33.8}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.954089999198914, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.13}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94830019958317, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.11}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11724211461842, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.25}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72319697216153, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 33.88}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85268899425864, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.1}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02760115265846, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.15}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00317093171179, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.03}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.61438589915633, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 33.83}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32027509249747, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.5}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96026001125574, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.08}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88881895504892, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.0}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59934597089887, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 33.89}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01025107875466, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.15}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77005716599524, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 33.79}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52888513170183, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 33.76}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.336933029815555, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 33.59}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68009699322283, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 33.92}], "seconds": 5.4, "failures": 0}, "reversed": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.50558507069945, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 33.75}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.988249979913235, "probs": [0.02983410046106534, 0.02983410046106534, 0.9403317990778695], "top": "supported", "option_logits": null, "prompt_sha256": "44861e558c186b2b16dcf17748026cc812485bd237c1ed73b0e7c6afc6a851c9", "input_tokens": 202, "server_ms": 34.26}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89882904291153, "probs": [0.0360284403416626, 0.9407485455412432, 0.023223014117094116], "top": "supported", "option_logits": null, "prompt_sha256": "09ba568fa9605e96b8b4411a9ca3aae662f9ea4803bb165ee522944d67fb01f7", "input_tokens": 196, "server_ms": 34.03}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.755128065124154, "probs": [0.20906175007679842, 0.01029042356271627, 0.7806478263604854], "top": "insufficient", "option_logits": null, "prompt_sha256": "cf3bcade5ac0e68f64b9e3de92a3184b942d098eaacb9f24e3300b6b94598271", "input_tokens": 196, "server_ms": 33.95}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80163808166981, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 33.94}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949398927390575, "probs": [0.9052958620137371, 0.016330598826197848, 0.07837353916006504], "top": "contradicted", "option_logits": null, "prompt_sha256": "488e5a4d1a83bc993ce696396f30450bd15bb21481f5518dc951c048237b93f8", "input_tokens": 206, "server_ms": 34.03}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.019341019913554, "probs": [0.03521557786284803, 0.04526073598298664, 0.9195236861541654], "top": "contradicted", "option_logits": null, "prompt_sha256": "fd494f61ffab19d399e8fb90311c05082df25926fb343185a8e2a5c0ef45865c", "input_tokens": 208, "server_ms": 34.17}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2831542044878, "probs": [0.956488422228046, 0.025140357850740234, 0.01837121992121384], "top": "insufficient", "option_logits": null, "prompt_sha256": "4556774746f89d946c8d41b92b4aadc4c1dd8ad7aec91836e3576a42b46e5d80", "input_tokens": 200, "server_ms": 34.46}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77949798293412, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 33.8}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16783178970218, "probs": [0.030254731557570292, 0.016155721873385053, 0.9535895465690447], "top": "supported", "option_logits": null, "prompt_sha256": "ef4c373b1ea1c02654d9f6861502d3e48140acc6e921812105a537a7306af012", "input_tokens": 194, "server_ms": 34.29}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97976004332304, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "9a355bc7ff84249e36d0f77394943b3c2c2c55e930cfac4ab5ac607aff2f52ef", "input_tokens": 193, "server_ms": 34.24}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05760114826262, "probs": [0.06594123405239463, 0.9194290330499963, 0.014629732897608938], "top": "insufficient", "option_logits": null, "prompt_sha256": "802949693168d625f8a46ddd7cbf0664d1582312a4dfc3600136983f6f0cc525", "input_tokens": 192, "server_ms": 34.26}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64889605715871, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 33.69}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.250613095238805, "probs": [0.934797668143608, 0.031578838206113354, 0.03362349365027861], "top": "contradicted", "option_logits": null, "prompt_sha256": "eb09cab56420a01e12c591b8cd5604aaae564c91d1eaca4f9f41ad0fd5b5233d", "input_tokens": 205, "server_ms": 34.36}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63256601430476, "probs": [0.9662013384858644, 0.017429270244818, 0.01636939126931757], "top": "contradicted", "option_logits": null, "prompt_sha256": "ca7284b45d19cedd96649c7650fd2bc131a85aa7b67f347b312ffeddeb465811", "input_tokens": 199, "server_ms": 33.81}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15607197955251, "probs": [0.0494468893746347, 0.007071172681351224, 0.943481937944014], "top": "insufficient", "option_logits": null, "prompt_sha256": "0e8c3b5d6ee999770b09a8d41f121d17841a8a3c88f99ca31776d98e3cb9a396", "input_tokens": 194, "server_ms": 34.36}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8051880504936, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 33.88}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00330015271902, "probs": [0.03400516513017273, 0.94540886408499, 0.020585970784837124], "top": "supported", "option_logits": null, "prompt_sha256": "c091f0fb6754104f2838a339bd3f3b7c66ba89f33bb05f47107decd9d42301c7", "input_tokens": 198, "server_ms": 34.27}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94346989504993, "probs": [0.03851217176175775, 0.9444509127063415, 0.017036915531900693], "top": "supported", "option_logits": null, "prompt_sha256": "498ed610492bb0a308366b9a8de1475f1d8a540b55af4127e4778ae21b45edbc", "input_tokens": 194, "server_ms": 34.17}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05019108019769, "probs": [0.9707570026002149, 0.016446573157518886, 0.012796424242266244], "top": "insufficient", "option_logits": null, "prompt_sha256": "260057e3eb2abfe4ed09b701c6deff3fb3715fdb0c8ca324a1348dfd774f2c82", "input_tokens": 194, "server_ms": 34.15}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.888089030981064, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.05}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04088111408055, "probs": [0.04850906601510146, 0.9255875820710483, 0.025903351913850195], "top": "contradicted", "option_logits": null, "prompt_sha256": "c9174dbf9bd60d78d5e86b6a48cf9c0e051be2112af8d047030597cc3117dbc8", "input_tokens": 211, "server_ms": 34.35}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.752486834302545, "probs": [0.03386081007082737, 0.9413955327676918, 0.024743657161480793], "top": "contradicted", "option_logits": null, "prompt_sha256": "d475c03e8dd15fbc64354f2d0129fbd69e2b3ba4f1e325567f185dead7f95af1", "input_tokens": 201, "server_ms": 33.92}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.365535041317344, "probs": [0.03868965206515757, 0.012507033178832465, 0.94880331475601], "top": "insufficient", "option_logits": null, "prompt_sha256": "6436470d3b6ae6205cb270b45c6c3d7ce3981366a138412b215b746633d85d03", "input_tokens": 197, "server_ms": 34.51}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92596894688904, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 34.18}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64268599823117, "probs": [0.03448559198311794, 0.9587656533150248, 0.006748754701857216], "top": "permitted", "option_logits": null, "prompt_sha256": "2b94c94ac232ec104d6110952de80a76a00fe63134632daf52ba9abf7e6b6c98", "input_tokens": 218, "server_ms": 33.82}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86326905153692, "probs": [0.9014745524219904, 0.07329691914188313, 0.025228528436126444], "top": "permitted", "option_logits": null, "prompt_sha256": "dba7aef8f49c04c2fdddde2913d398830805d2a0db86474bf3980f131ff38f40", "input_tokens": 203, "server_ms": 33.91}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23750286735594, "probs": [0.07600853330506088, 0.046013876624721124, 0.8779775900702179], "top": "insufficient", "option_logits": null, "prompt_sha256": "ef6236452d6617250251cedd641ebfc6643ccf809d4ac681657e917ec058827e", "input_tokens": 209, "server_ms": 34.43}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75373690202832, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.06}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39272500388324, "probs": [0.07865862910342072, 0.9085889611870406, 0.012752409709538734], "top": "prohibited", "option_logits": null, "prompt_sha256": "6877f3535caa9f30ae266f09910684ff8191f357e26f28235801e3ee61d863c6", "input_tokens": 214, "server_ms": 34.52}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02802094630897, "probs": [0.3226628352473404, 0.4415525480598312, 0.23578461669282835], "top": "insufficient", "option_logits": null, "prompt_sha256": "84f9c8de74a885791a9adb02311064234d0f3c511ab5d8bde62bc920f6c9ad41", "input_tokens": 209, "server_ms": 34.17}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10605204105377, "probs": [0.11063351808551876, 0.8770566410678413, 0.012309840846639953], "top": "insufficient", "option_logits": null, "prompt_sha256": "990c725e7e508f4293dd16353f1b16e7bb890bdc1bf5f3d13a693d4e617635c6", "input_tokens": 209, "server_ms": 34.37}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99568985030055, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.25}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53274609334767, "probs": [0.05501866755885692, 0.07529107902609294, 0.8696902534150501], "top": "permitted", "option_logits": null, "prompt_sha256": "6e590e2135227d5cca28d2e60e24fd54e93c992a155bfc23179ee4c580915112", "input_tokens": 211, "server_ms": 34.28}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02952107414603, "probs": [0.943076329528072, 0.040946062905879925, 0.01597760756604808], "top": "permitted", "option_logits": null, "prompt_sha256": "f0a55e8157d1588348f58d9d803f9bac80b18f2c0cd81e8afbf5f86cc0bc54fa", "input_tokens": 202, "server_ms": 34.16}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03393111936748, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "5fe8edf564c5caecc2816e1e5b24dc2e11e7e49716f15277d0ad87e04fb0441e", "input_tokens": 204, "server_ms": 34.12}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00498095713556, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.07}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18409198336303, "probs": [0.08617432324783239, 0.8780222847407805, 0.03580339201138707], "top": "prohibited", "option_logits": null, "prompt_sha256": "de7b2b0f8c3b1dea2ae34d68854d8932985f8194c5d9ef94ef9ac3c5af3abf22", "input_tokens": 214, "server_ms": 34.47}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95817990228534, "probs": [0.060153978502511585, 0.046803417311243246, 0.8930426041862451], "top": "prohibited", "option_logits": null, "prompt_sha256": "c58dd6040c3a5d9472ff44ab8e41e9eb92a7c0011e1b1ba9021afcf2d6b1a11a", "input_tokens": 213, "server_ms": 34.2}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.199333077296615, "probs": [0.3370535895810829, 0.14003778663063773, 0.5229086237882794], "top": "insufficient", "option_logits": null, "prompt_sha256": "eb881b14aecfc2220563d1a2f324b20bccfcfe0fd127368ba2a8ed32ee653ac1", "input_tokens": 212, "server_ms": 34.37}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.131962133571506, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 34.24}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08181204274297, "probs": [0.043034915318676176, 0.9309127048683882, 0.026052379812935716], "top": "permitted", "option_logits": null, "prompt_sha256": "ef8d68233fa0d3bb7526e3c22aa69a27e6f6fbda1b973de57cbe90e38379d4c0", "input_tokens": 216, "server_ms": 34.19}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95017006248236, "probs": [0.9036036371977804, 0.039232255151330435, 0.057164107650889244], "top": "permitted", "option_logits": null, "prompt_sha256": "5f63161d15c1144a576bb01894090a0d08c4f3bfdcfe5fc99192ebe00be5a0ec", "input_tokens": 208, "server_ms": 33.96}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.471395917236805, "probs": [0.8930317651393958, 0.06819498463159931, 0.03877325022900488], "top": "insufficient", "option_logits": null, "prompt_sha256": "98dbd799628876f678d1de46679eb443541ecb76f026ad2b8b781efa1705880a", "input_tokens": 215, "server_ms": 34.6}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87630989588797, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 34.1}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96062904782593, "probs": [0.42551441767130554, 0.06085086611336375, 0.5136347162153307], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ae6464839efd58c0371c609fe15542b4468085854cec4634bb01da9bb661271", "input_tokens": 212, "server_ms": 33.91}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73633816652, "probs": [0.2363974954976129, 0.6450449475319231, 0.11855755697046397], "top": "prohibited", "option_logits": null, "prompt_sha256": "1ad4c75b16b0473861999a960bca9c7cce8eac322d72f239942517d07ecc7531", "input_tokens": 209, "server_ms": 33.78}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12441283278167, "probs": [0.5169331333226727, 0.45597494786051085, 0.02709191881681649], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6546fb5bf3590e9fba39e6d78cfd36e6155363cfafeb691260ad754365b88a2", "input_tokens": 210, "server_ms": 34.11}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81335807777941, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 33.93}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98463994078338, "probs": [0.03143752450054959, 0.03794796085693116, 0.9306145146425192], "top": "B", "option_logits": null, "prompt_sha256": "d3eb956ec7e3e32cf66ce5d717522d8d92b5dac4c8308f4c9d4cb31fe756303e", "input_tokens": 201, "server_ms": 33.97}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63163702003658, "probs": [0.08136038144369948, 0.8826476192952379, 0.03599199926106259], "top": "B", "option_logits": null, "prompt_sha256": "cd399d53be08dd01e71ea966eaf17f42207db08d1eeeef6472c99f8f8fe53f13", "input_tokens": 192, "server_ms": 33.67}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8489190004766, "probs": [0.10525990911259618, 0.888486034617855, 0.00625405626954877], "top": "A", "option_logits": null, "prompt_sha256": "eb3cee4bfe759b0bc88ef31833cbc8682cc73ee0d02c2a21d2d3ad10a8b33f3d", "input_tokens": 192, "server_ms": 33.97}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66185587644577, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 33.73}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35582483932376, "probs": [0.04147910382575712, 0.95535340876032, 0.003167487413922894], "top": "A", "option_logits": null, "prompt_sha256": "6305741f94289ba3f3621c2b6eb0f190d9314fd35bf4428bc6faee6c58477958", "input_tokens": 211, "server_ms": 34.3}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.683195969089866, "probs": [0.9728143992091232, 0.0198945881954946, 0.007291012595382136], "top": "A", "option_logits": null, "prompt_sha256": "54e77d8b88f6d4d5ee3cfd402bbd6f306bf44e79427bd574d63cd3f4cfb05e72", "input_tokens": 200, "server_ms": 33.76}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32445416972041, "probs": [0.3414721146107966, 0.6394729578033058, 0.019054927585897746], "top": "A", "option_logits": null, "prompt_sha256": "5ace07508957ec619cb5e27ef2963d8fb638a1dbc11c9f5324862d847526978c", "input_tokens": 190, "server_ms": 34.52}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02903096564114, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.08}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17337199486792, "probs": [0.019333349474436403, 0.0927842897505625, 0.887882360775001], "top": "B", "option_logits": null, "prompt_sha256": "0693c147f7e7d616b5187d9ab344a3da9ce0775cfe9c7bbbb763e22f35e065ba", "input_tokens": 206, "server_ms": 34.19}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.49607487022877, "probs": [0.03036360301441411, 0.9570210386636204, 0.012615358321965421], "top": "B", "option_logits": null, "prompt_sha256": "18217c60f50c0d632786b29cb4d780e7e97192fb8f872b0f1756a8ffa38a2edc", "input_tokens": 197, "server_ms": 33.62}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99824004434049, "probs": [0.1762660158495679, 0.1655472114938264, 0.6581867726566057], "top": "insufficient", "option_logits": null, "prompt_sha256": "688e414d6cdfa5a61e0f0668cb9967fd977b1f20a95cfe725e9bc169ed05549c", "input_tokens": 192, "server_ms": 34.11}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73004708252847, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 33.77}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83518897742033, "probs": [0.05076536678270011, 0.039498512654509606, 0.9097361205627902], "top": "A", "option_logits": null, "prompt_sha256": "882794cff48da2b9528a41f2219d35fab11c254f26ff72c815aa40552eb41e83", "input_tokens": 203, "server_ms": 33.94}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85593884438276, "probs": [0.0494468893746347, 0.943481937944014, 0.007071172681351224], "top": "A", "option_logits": null, "prompt_sha256": "1325099ea764f8c6251ce0c006048d9276e68171e27006136ee3aa20f11a03e8", "input_tokens": 193, "server_ms": 34.05}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88656911253929, "probs": [0.8780235296813632, 0.10402040687475302, 0.01795606344388373], "top": "insufficient", "option_logits": null, "prompt_sha256": "a613883b12602a7fb71d9d55a911f2614490592581d6471116c4d4c68a4c8957", "input_tokens": 192, "server_ms": 33.99}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91007988527417, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 33.88}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.867169197648764, "probs": [0.9647591205470982, 0.03259097850339806, 0.0026499009495037375], "top": "B", "option_logits": null, "prompt_sha256": "2b94950be2ada8957f3d093c3101d1935356aed904ff6f32af5e780ca4daf0fa", "input_tokens": 206, "server_ms": 33.94}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.587066017091274, "probs": [0.900473990544217, 0.08837786536850668, 0.01114814408727632], "top": "B", "option_logits": null, "prompt_sha256": "98330242bf9b4853692144bd273028a1065e63fecbd536b99f72041a124cbdde", "input_tokens": 200, "server_ms": 33.58}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09586104378104, "probs": [0.46462686920294194, 0.07074626159411614, 0.46462686920294194], "top": "A", "option_logits": null, "prompt_sha256": "0a00b1c54151f720e7eb71eb04001a6783321bfab6008e2e1e9749ec127ea479", "input_tokens": 193, "server_ms": 34.24}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.736497888341546, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 33.74}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8585588876158, "probs": [0.05584400041528186, 0.9398915591084774, 0.004264440476240678], "top": "A", "option_logits": null, "prompt_sha256": "104d6ac93e38d37c2cbd2fe1d413e52ca92dde1258cac35fc220528b73196b21", "input_tokens": 209, "server_ms": 33.87}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78372781723738, "probs": [0.04937458683512937, 0.008523069447562875, 0.9421023437173077], "top": "A", "option_logits": null, "prompt_sha256": "d217d38b7e6213bb691099e21c66a3df1e10fe6cde75e36ed398dd45fd11b8e9", "input_tokens": 194, "server_ms": 33.94}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73405712470412, "probs": [0.19870361005143694, 0.18662038312341964, 0.6146760068251435], "top": "insufficient", "option_logits": null, "prompt_sha256": "ee048efb2e813aef04a41f981140fbeb7ab7de14a0a4baac09c4a808bb63510f", "input_tokens": 192, "server_ms": 33.89}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63513599894941, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 33.77}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10176213644445, "probs": [0.8170038444540815, 0.05859532327955457, 0.12440083226636395], "top": "supported", "option_logits": null, "prompt_sha256": "17fe8aeab450890957725a1dad43750752505b560e3976c468dc3f41172f8a67", "input_tokens": 214, "server_ms": 34.15}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66036599315703, "probs": [0.015242642754936796, 0.026808995604051852, 0.9579483616410114], "top": "supported", "option_logits": null, "prompt_sha256": "9e0440adb82ccd063e1c322dc6a4b4897225905d32a3f7f1bb9049191611cc67", "input_tokens": 209, "server_ms": 33.76}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.837318912148476, "probs": [0.05587209717642955, 0.9403644443692992, 0.0037634584542711896], "top": "insufficient", "option_logits": null, "prompt_sha256": "d0c71930cb8f313e647a558d26ffefabb510d65a93a3f32e41872606cc6bf95b", "input_tokens": 207, "server_ms": 33.92}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59356595017016, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 33.73}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.930349888280034, "probs": [0.28553873616003933, 0.3237117094730352, 0.39074955436692543], "top": "permitted", "option_logits": null, "prompt_sha256": "daa6ab1802b1fde61e5a315b23c0a03e284c6fadfbc803e995b1b2f126db4ff1", "input_tokens": 229, "server_ms": 33.96}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.926029017195106, "probs": [0.9139932977706335, 0.05100292527189731, 0.035003776957469146], "top": "permitted", "option_logits": null, "prompt_sha256": "bcfabc366ceb030ad33c3667e455143117c067135419a04c514f9eedf56beef4", "input_tokens": 222, "server_ms": 33.97}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.694857059046626, "probs": [0.055084494859207855, 0.017806923046620855, 0.9271085820941714], "top": "insufficient", "option_logits": null, "prompt_sha256": "4648f72540215e378c489522ed8bd6dc3c3d325e632d3d70fb9169a60a51ca09", "input_tokens": 221, "server_ms": 33.85}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.587256006896496, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.74}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.081240909174085, "probs": [0.04144166000042449, 0.9544910095891707, 0.004067330410404966], "top": "A", "option_logits": null, "prompt_sha256": "2298533e13693dfd7fd2fbb3c3e005385ef1d63a9b81983a9d3e25feb90ff23c", "input_tokens": 210, "server_ms": 34.09}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80454799905419, "probs": [0.6989250414551824, 0.06050754282191963, 0.240567415722898], "top": "A", "option_logits": null, "prompt_sha256": "c9d6f93bbd17ec079e798d376df024790d2bfa77871b44e70fd8616a0ad58211", "input_tokens": 209, "server_ms": 33.86}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88337979838252, "probs": [0.9290594611076342, 0.055200406122332214, 0.015740132770033617], "top": "insufficient", "option_logits": null, "prompt_sha256": "e508f7c0cf5b6f7bf721fbea254629adeb5723dd71b126c9dad2526fd6478e69", "input_tokens": 211, "server_ms": 34.03}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80864814668894, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 33.92}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07842202857137, "probs": [0.7687919368693059, 0.037841427439577066, 0.19336663569111703], "top": "supported", "option_logits": null, "prompt_sha256": "2576bc0fe1d0394c448cc1d2e3cd03c33f1d7a6a8b5f8c0c267ac79d4ecbb2e0", "input_tokens": 227, "server_ms": 34.04}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8167778942734, "probs": [0.028437503812587046, 0.01721543167027419, 0.9543470645171389], "top": "supported", "option_logits": null, "prompt_sha256": "c88d87385c3f5978b7b5ff8266b180fb20725224ea5035ed6f7e759ad7175b45", "input_tokens": 202, "server_ms": 33.87}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05352102406323, "probs": [0.03842465563368609, 0.01927064844887017, 0.9423046959174438], "top": "insufficient", "option_logits": null, "prompt_sha256": "5d00998ccf948a378fe3730d22365cc93d6ca3447a6bffab9c27db9ba203c73f", "input_tokens": 202, "server_ms": 34.18}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.637316923588514, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 33.64}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00918983668089, "probs": [0.4756119457847697, 0.27041607197430173, 0.2539719822409286], "top": "insufficient", "option_logits": null, "prompt_sha256": "f26149405c562e81cb9ef93186374e44a3d38ec943d4bc1328305de7bfd2f3c5", "input_tokens": 231, "server_ms": 34.0}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68760717846453, "probs": [0.8133507124046239, 0.05478605406972841, 0.13186323352564766], "top": "permitted", "option_logits": null, "prompt_sha256": "bcca8c96798eb11f3f4644bd6b48d9956d0ef797acb354a9d313e72d7decc3e6", "input_tokens": 216, "server_ms": 33.73}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97484009899199, "probs": [0.17523657347175897, 0.6967099745500006, 0.1280534519782404], "top": "insufficient", "option_logits": null, "prompt_sha256": "39afa3600a44ace025f89dc8a73ad8b6ad0761af51bdcb709832b17c14127f54", "input_tokens": 223, "server_ms": 34.17}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.554815946146846, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 33.79}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89967910759151, "probs": [0.9147597146359207, 0.07919284608441045, 0.006047439279669017], "top": "B", "option_logits": null, "prompt_sha256": "c4b6e37f55e9e861c7a6f927859de07ee371fe7aca51ca2ae85600e1cd82765e", "input_tokens": 222, "server_ms": 33.97}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79438796825707, "probs": [0.8768319468998427, 0.09162951349252214, 0.03153853960763513], "top": "B", "option_logits": null, "prompt_sha256": "83a38a50ea29d93de3d3e3463c307552985b65ef7fbecfb33f19f0c8b19cbe78", "input_tokens": 215, "server_ms": 33.88}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28946391493082, "probs": [0.9751721172869884, 0.022608910103960853, 0.0022189726090507235], "top": "insufficient", "option_logits": null, "prompt_sha256": "65c35f2d10deb3715357e389ebf9ee5a130cde3069dda04db44dcb11491b8adc", "input_tokens": 201, "server_ms": 34.49}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.605386063456535, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 33.72}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93212987855077, "probs": [0.08075843891914115, 0.04312419313814581, 0.876117367942713], "top": "supported", "option_logits": null, "prompt_sha256": "4f8bbbb0dfd533b6deedfbbd1466a23c3b560303a1f961ed21be386fa842301a", "input_tokens": 219, "server_ms": 34.08}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5872650872916, "probs": [0.9535895465690447, 0.016155721873385053, 0.030254731557570292], "top": "supported", "option_logits": null, "prompt_sha256": "c61081105c19ff493658ba57f096ae6e9380cb6cb3a7cdc1596f4c36991b4710", "input_tokens": 220, "server_ms": 33.84}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92282899282873, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "insufficient", "option_logits": null, "prompt_sha256": "94d9b1e865081ca9ee6a9204c96cb1afbd9655d3298d6fd5bdca85429f6d7531", "input_tokens": 218, "server_ms": 34.0}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67107690125704, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 33.91}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60746617242694, "probs": [0.47103444559681956, 0.08657488347540951, 0.442390670927771], "top": "prohibited", "option_logits": null, "prompt_sha256": "0984ef33039be3140b3da26a6910b546b1f605153e5b197341ffac510f80627a", "input_tokens": 221, "server_ms": 33.75}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.594856064766645, "probs": [0.049073786622505555, 0.0715038981480174, 0.879422315229477], "top": "permitted", "option_logits": null, "prompt_sha256": "cd350bbed40603bfdec8aee19ed8f4aead08aae6c3e75e5950990320c9a1602f", "input_tokens": 221, "server_ms": 33.68}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77833708934486, "probs": [0.039087581768329945, 0.9002715143788037, 0.06064090385286648], "top": "insufficient", "option_logits": null, "prompt_sha256": "32a2ba65d112e44959b5167505e017413199fda0540cd2a995538aa795dabd11", "input_tokens": 220, "server_ms": 33.99}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67535702697933, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 33.81}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90140903927386, "probs": [0.02394587818769938, 0.9700312518040514, 0.006022870008249252], "top": "B", "option_logits": null, "prompt_sha256": "0d78a5511291c0e8891da715a82ec2ad438e94f5dddaee4ff0a75c528e00ad7e", "input_tokens": 230, "server_ms": 33.93}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.745646968483925, "probs": [0.028667490838461873, 0.9620652951327192, 0.00926721402881876], "top": "B", "option_logits": null, "prompt_sha256": "f925b35d49f8ca5aac7190490beee3fe1f89e7ea2183f46b6ff8de5291cc99d6", "input_tokens": 223, "server_ms": 33.73}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18785289674997, "probs": [0.018723542031095865, 0.9748318841721378, 0.0064445737967663164], "top": "insufficient", "option_logits": null, "prompt_sha256": "6c61299dde52e0bf1bae5a513eccf6464ab11fd42eab81e0c811d717c5ee78fe", "input_tokens": 210, "server_ms": 34.17}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72432713210583, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 33.83}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91586898639798, "probs": [0.8748952051681804, 0.027757979619760474, 0.09734681521205904], "top": "supported", "option_logits": null, "prompt_sha256": "b21acb3da5a82760fd02a24aed0fc2babe9ffe3bc13c3b214a041901cb3e0d7d", "input_tokens": 224, "server_ms": 33.88}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69967711903155, "probs": [0.07897871706298064, 0.6261104979730077, 0.29491078496401174], "top": "insufficient", "option_logits": null, "prompt_sha256": "11a658148b97ed6304e23da68b8382fe860758bceb2aab62101a22fb8e2a2d9b", "input_tokens": 220, "server_ms": 33.72}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37743501551449, "probs": [0.9241406947470572, 0.024290130010671554, 0.051569175242271295], "top": "insufficient", "option_logits": null, "prompt_sha256": "d6185ec81e36298077e1827a7b4a5e68c397cf8dde6ebe5e37e25b43ca01882d", "input_tokens": 208, "server_ms": 34.58}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.51989484205842, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 33.65}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11258200928569, "probs": [0.21207317204418882, 0.04418942927644211, 0.743737398679369], "top": "insufficient", "option_logits": null, "prompt_sha256": "529e850d9f1fbc6eacd215e95ebfbc941317a3f5224bff5d7dea44065cd2e1a9", "input_tokens": 228, "server_ms": 34.34}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95992985740304, "probs": [0.045449474437266706, 0.031192392974493464, 0.9233581325882398], "top": "permitted", "option_logits": null, "prompt_sha256": "81df99f7f22c5619613f932264037a494407bd0c9c1cd772d88cdb13bd93f038", "input_tokens": 225, "server_ms": 34.11}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86931892298162, "probs": [0.9572108985191267, 0.032335988189970806, 0.010453113290902514], "top": "insufficient", "option_logits": null, "prompt_sha256": "737e84a1e8c9bcc630c5ec500de2a3e5db92ca702b06dcd3ad924711e2fbdbc3", "input_tokens": 218, "server_ms": 34.03}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.897170124575496, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.13}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00027987360954, "probs": [0.37235037560509904, 0.615071082756213, 0.012578541638687917], "top": "A", "option_logits": null, "prompt_sha256": "c5cb029101b9f71556e6e018776a50195e4ada1ad92940e55665671b3868297a", "input_tokens": 227, "server_ms": 34.29}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.874859826639295, "probs": [0.12177174509735464, 0.07849096626584667, 0.7997372886367987], "top": "A", "option_logits": null, "prompt_sha256": "dd57bc49c9dda9182642dd072b29b16671a713f0b0824c487439f9e2b331573e", "input_tokens": 207, "server_ms": 34.1}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83684905990958, "probs": [0.073881614277135, 0.017452691866116716, 0.9086656938567483], "top": "insufficient", "option_logits": null, "prompt_sha256": "bda0e38b4228f4220bca836f2281dd0b297cef74516258475f031ebc54637c78", "input_tokens": 197, "server_ms": 34.09}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0304009411484, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 34.11}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78002906963229, "probs": [0.9825585277035805, 0.01007742973075826, 0.007364042565661154], "top": "supported", "option_logits": null, "prompt_sha256": "cc5454a07757b53ce2f838d93f3f4c26856cbc40d4fdb7c2edb21f43da36a5cc", "input_tokens": 212, "server_ms": 33.77}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.886079935356975, "probs": [0.02825825432560628, 0.023410205377014614, 0.9483315402973791], "top": "supported", "option_logits": null, "prompt_sha256": "b1739f17dac9c6d981d59ce4523ac7852dffdb509918039bd8d833acb430ac6c", "input_tokens": 202, "server_ms": 34.1}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66386706754565, "probs": [0.6892396519371748, 0.005856275914839899, 0.30490407214798526], "top": "insufficient", "option_logits": null, "prompt_sha256": "98f4588180d28f5dcef4c3bf07ecf8e021f406ba222ff13a774d337df06379ee", "input_tokens": 203, "server_ms": 33.93}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.591496085748076, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 33.83}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1091419365257, "probs": [0.036422612390539545, 0.9510408592905502, 0.012536528318910257], "top": "permitted", "option_logits": null, "prompt_sha256": "1e8882942aa8fcb63c49bda41019547d5c388d1eb722d8f3d2f6ff0fe2f6de83", "input_tokens": 221, "server_ms": 34.34}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70221706852317, "probs": [0.03221192217085971, 0.014249827577513809, 0.9535382502516264], "top": "permitted", "option_logits": null, "prompt_sha256": "7ab28da3bc25386c1a377fd6ee20ae9c0e59159093b9600f9039749931b1d405", "input_tokens": 215, "server_ms": 34.0}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35963487997651, "probs": [0.6096504219680668, 0.3466255945273742, 0.043723983504558944], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce7c09276f0384e8534c52aa20c9df30f1b7a23977287fb0373141421ede2323", "input_tokens": 220, "server_ms": 34.54}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69748711399734, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 33.78}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09453111514449, "probs": [0.026052379812935716, 0.043034915318676176, 0.9309127048683882], "top": "B", "option_logits": null, "prompt_sha256": "3e92d9a3ec71acb771522ae129c59a1c3f3cf135471b1a9119034e8e60f57534", "input_tokens": 212, "server_ms": 34.21}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.475355038419366, "probs": [0.010960515985196186, 0.046398608171146855, 0.942640875843657], "top": "B", "option_logits": null, "prompt_sha256": "6c5f599d35035dfb7793d063356814cf6c7a52ffa3dd47f92c4bf500ca7dfd33", "input_tokens": 203, "server_ms": 33.7}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16649208217859, "probs": [0.039029590587771126, 0.9571398140015086, 0.0038305954107202447], "top": "insufficient", "option_logits": null, "prompt_sha256": "37218ff944f5d39cce6127759ca5add34909346542a0085ae71b27113a5d94dc", "input_tokens": 191, "server_ms": 34.3}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84180788695812, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.03}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27760398760438, "probs": [0.06296298480307037, 0.877902829429632, 0.05913418576729761], "top": "supported", "option_logits": null, "prompt_sha256": "ffb22e022a23a832611c31cce83e5ded9325f32d02e4447965afe9987da2bde6", "input_tokens": 221, "server_ms": 34.45}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.50628495961428, "probs": [0.9514443415619939, 0.009758312735022333, 0.038797345702983814], "top": "supported", "option_logits": null, "prompt_sha256": "7a3eec9452b83ccbec55aa1300b3feef68ed5e14955200c3b0e7ea8e89a95d42", "input_tokens": 229, "server_ms": 33.72}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.026230012997985, "probs": [0.026899996114941614, 0.011899951244465976, 0.9612000526405924], "top": "insufficient", "option_logits": null, "prompt_sha256": "6210dc877e4c1f0e5a92c78000354ff443f3add66bc382cd9fc3184bc1cde515", "input_tokens": 226, "server_ms": 34.22}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.816797917708755, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.07}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93644003942609, "probs": [0.12359582333016605, 0.8642737486438103, 0.01213042802602361], "top": "permitted", "option_logits": null, "prompt_sha256": "b064243d3870600f856ee53a1ef805a900c11107d12f38f2c3670753f532aff4", "input_tokens": 215, "server_ms": 34.0}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77120804786682, "probs": [0.20386035032714242, 0.5922792993457151, 0.20386035032714242], "top": "permitted", "option_logits": null, "prompt_sha256": "d0e6877a0a8a83928384752e861479053fa3570f13d19a5a7be7d8ed7368055d", "input_tokens": 225, "server_ms": 33.95}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.060532020404935, "probs": [0.4345988057387763, 0.10266314506169773, 0.462738049199526], "top": "insufficient", "option_logits": null, "prompt_sha256": "f8d2b6eea2aa49d96d8da956f53d998cbb5844a3b8354672ea7172d7f3564f75", "input_tokens": 224, "server_ms": 34.12}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74906678497791, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 33.8}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.021570140495896, "probs": [0.15827071505004672, 0.808750606384142, 0.032978678565811305], "top": "B", "option_logits": null, "prompt_sha256": "d0cfebf59620575e28c1572d9cd5fe9a5f21b04d6f113348f66e043ebd3db047", "input_tokens": 218, "server_ms": 34.27}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5406759083271, "probs": [0.8012135216734096, 0.1567957257725581, 0.04199075255403233], "top": "B", "option_logits": null, "prompt_sha256": "7cb2185c68200e9ee262ca2de7508a98cc4dcde47a5db7a8e9587bc9489b12cf", "input_tokens": 209, "server_ms": 33.68}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3110449872911, "probs": [0.06962296989463873, 0.018645411826949065, 0.9117316182784122], "top": "insufficient", "option_logits": null, "prompt_sha256": "cfa1a906fceb6b3b9d7b5c051d4c802da2bb13a38396312d0a36f9d579150568", "input_tokens": 207, "server_ms": 34.55}], "seconds": 5.3, "failures": 0}, "shuffled": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97729995474219, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.16}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82745900005102, "probs": [0.9674618846083878, 0.012752988686722479, 0.01978512670488969], "top": "supported", "option_logits": null, "prompt_sha256": "eec32c94bdef649fc890b55ec49cf1aa2f19a0462a3385eb89ed703a10ecac10", "input_tokens": 202, "server_ms": 34.04}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.986259976401925, "probs": [0.03821459235254428, 0.024632151448974626, 0.9371532561984811], "top": "supported", "option_logits": null, "prompt_sha256": "48ceb67afca89ff6de3c590c914dcb0fd4b9e8e4348d19e33d11bcddfee33702", "input_tokens": 196, "server_ms": 34.23}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72885708510876, "probs": [0.20906175007679842, 0.01029042356271627, 0.7806478263604854], "top": "insufficient", "option_logits": null, "prompt_sha256": "cf3bcade5ac0e68f64b9e3de92a3184b942d098eaacb9f24e3300b6b94598271", "input_tokens": 196, "server_ms": 33.87}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.45089501515031, "probs": [0.028546574507428056, 0.01344601762615991, 0.958007407866412], "top": "supported", "option_logits": null, "prompt_sha256": "55d43d0de49c72c7c509fc43c61179257e038a6cfe16e115f6a2a0e587b33824", "input_tokens": 209, "server_ms": 33.65}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.711127031594515, "probs": [0.07810937869521252, 0.9022445396051365, 0.019646081699650965], "top": "contradicted", "option_logits": null, "prompt_sha256": "b45f2030e55f3fdf7179b1673f65e09b09bd2a5659a186837a6bb62fbd78bd00", "input_tokens": 206, "server_ms": 33.97}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78368800319731, "probs": [0.0464300360263431, 0.8859181944824366, 0.06765176949122025], "top": "contradicted", "option_logits": null, "prompt_sha256": "223a68676447bbcb6017f47a40a96cd18bed0fa7f075b38653f938db60b3baa9", "input_tokens": 208, "server_ms": 34.06}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91306011751294, "probs": [0.9436516791652002, 0.02640889059428092, 0.02993943024051884], "top": "insufficient", "option_logits": null, "prompt_sha256": "24d025485f5e4a2bff769117ed138515cf93200b06e742e455f7b8cc7152e3de", "input_tokens": 200, "server_ms": 34.04}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17418317683041, "probs": [0.9674618846083878, 0.01978512670488969, 0.012752988686722479], "top": "contradicted", "option_logits": null, "prompt_sha256": "b372c17c6c8de4fafff4d6c0ce468d3b54126dbfeef46d08db082bc829064884", "input_tokens": 194, "server_ms": 34.29}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.613915115594864, "probs": [0.02369913709056687, 0.016264936907900464, 0.9600359260015328], "top": "supported", "option_logits": null, "prompt_sha256": "57e9f770948d7778575f35784aa034b97ae3b3b2f2bd934c9e75f5fe7441659a", "input_tokens": 194, "server_ms": 33.84}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74385813064873, "probs": [0.9757912442213938, 0.012862785395550868, 0.011345970383055229], "top": "supported", "option_logits": null, "prompt_sha256": "61b6f283623ce18a30fbb0b6e030082367529d979b50606a19e5c27e971820ee", "input_tokens": 193, "server_ms": 33.99}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96302999742329, "probs": [0.055144207653558895, 0.01674220672178504, 0.928113585624656], "top": "insufficient", "option_logits": null, "prompt_sha256": "7cff73a57c8f079413f2a0b18374068ed86d94e2fafc91677c3742c14a066cab", "input_tokens": 192, "server_ms": 34.13}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.895519122481346, "probs": [0.028650300519022835, 0.009861327530892165, 0.9614883719500851], "top": "supported", "option_logits": null, "prompt_sha256": "7c45f0c0139d61a53e6208f0099673275733ff83d3f75da4dc4c78f43036b0f8", "input_tokens": 205, "server_ms": 34.02}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.133442936465144, "probs": [0.02907704833321378, 0.05445242977074118, 0.916470521896045], "top": "contradicted", "option_logits": null, "prompt_sha256": "50848dffd2c1c615f5b75f0bb9bd41d494049c38a0e68f53f8fb55948e5b8111", "input_tokens": 205, "server_ms": 34.37}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.51900589466095, "probs": [0.028121917950149154, 0.9437561640997016, 0.028121917950149154], "top": "contradicted", "option_logits": null, "prompt_sha256": "d9a00b5a305e35828cb17b95e58a1533b2089ed424d7bffc91ecfce3160c9611", "input_tokens": 199, "server_ms": 33.74}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19603200443089, "probs": [0.0494468893746347, 0.007071172681351224, 0.943481937944014], "top": "insufficient", "option_logits": null, "prompt_sha256": "0e8c3b5d6ee999770b09a8d41f121d17841a8a3c88f99ca31776d98e3cb9a396", "input_tokens": 194, "server_ms": 34.23}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61477814987302, "probs": [0.1014573935767339, 0.042153146641024326, 0.8563894597822418], "top": "contradicted", "option_logits": null, "prompt_sha256": "e924e4a25b3371462e128b92799f79eea841a96f2a2b5ef2d8d204f809b5e06b", "input_tokens": 198, "server_ms": 34.64}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43797494098544, "probs": [0.02825825432560628, 0.023410205377014614, 0.9483315402973791], "top": "supported", "option_logits": null, "prompt_sha256": "08da99ccc2e188d94eb1dd1b284d80a2f94349d15043417dadbf3ea7bb569d06", "input_tokens": 198, "server_ms": 34.63}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.681897938251495, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "supported", "option_logits": null, "prompt_sha256": "8408e99d3352a39e8e873a1c4cef9a62a8418f37bdff3e002eb1f6f2306fa17b", "input_tokens": 194, "server_ms": 33.72}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62721689417958, "probs": [0.9707570026002149, 0.016446573157518886, 0.012796424242266244], "top": "insufficient", "option_logits": null, "prompt_sha256": "260057e3eb2abfe4ed09b701c6deff3fb3715fdb0c8ca324a1348dfd774f2c82", "input_tokens": 194, "server_ms": 34.79}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.51413484476507, "probs": [0.027692497402509345, 0.04296244408302007, 0.9293450585144706], "top": "supported", "option_logits": null, "prompt_sha256": "4b7f6e13d64e3b44753244fe8bcc35e3c61bb5cbc56885204150c2b8a1d730f4", "input_tokens": 211, "server_ms": 33.7}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184292916208506, "probs": [0.038153743171136366, 0.026185265088641236, 0.9356609917402224], "top": "contradicted", "option_logits": null, "prompt_sha256": "a8983f75a788c808d83e49e31b2c7fc60f168203e3371a2f46e674b96cba586a", "input_tokens": 211, "server_ms": 34.29}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.407804979011416, "probs": [0.0382719206436522, 0.023168966945696274, 0.9385591124106515], "top": "contradicted", "option_logits": null, "prompt_sha256": "b0a3a7311091e1405f7995266a413ad483489714075309b06ec871e31f77954f", "input_tokens": 201, "server_ms": 34.57}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01812983490527, "probs": [0.03621509737243083, 0.9456223856962459, 0.01816251693132339], "top": "insufficient", "option_logits": null, "prompt_sha256": "da3839c491c29e9cc50a18a3a20d59e3f9ff7b0e96a001ae41c4bf65f5f0eab1", "input_tokens": 197, "server_ms": 34.12}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.237543845549226, "probs": [0.6993354607136348, 0.1872858543621493, 0.11337868492421596], "top": "prohibited", "option_logits": null, "prompt_sha256": "b4ef5af7abdfbfcdcb84a50cd028b197644864319b662c102080599d74c2a232", "input_tokens": 212, "server_ms": 34.4}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.496285047382116, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "permitted", "option_logits": null, "prompt_sha256": "b961f1c42a0a57ea80e203f2f69b241844174acdf3b8cfe84ce6ec3ffb79fc8c", "input_tokens": 218, "server_ms": 34.63}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6146070882678, "probs": [0.05035348378460727, 0.9023550434611196, 0.047291472754273096], "top": "permitted", "option_logits": null, "prompt_sha256": "742a7db029a78d04d26d019231b01690c69683a73384a8da6aa929cd15c81513", "input_tokens": 203, "server_ms": 33.86}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97815001942217, "probs": [0.09241428693559126, 0.8843416612729131, 0.023244051791495635], "top": "insufficient", "option_logits": null, "prompt_sha256": "094c6ca6168a09f4dc1447959e11a10d6a06018c0e4cfebb716806ee019cbfc0", "input_tokens": 209, "server_ms": 34.05}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.47312382236123, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 33.65}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69907711446285, "probs": [0.9137406766907489, 0.06977638840966881, 0.016482934899582382], "top": "prohibited", "option_logits": null, "prompt_sha256": "c831b10f220597e08250d09cef04d9dcbbd20d23498389b2ec05cf891745d2e4", "input_tokens": 214, "server_ms": 33.93}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.54995490796864, "probs": [0.3226628352473404, 0.4415525480598312, 0.23578461669282835], "top": "insufficient", "option_logits": null, "prompt_sha256": "84f9c8de74a885791a9adb02311064234d0f3c511ab5d8bde62bc920f6c9ad41", "input_tokens": 209, "server_ms": 33.78}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66248684749007, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "171ddb6e8d557e07c9968a706d339748b14038c7411892826fa822fb48e1baf6", "input_tokens": 209, "server_ms": 33.93}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85483802109957, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 33.88}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9497099891305, "probs": [0.05501866755885692, 0.07529107902609294, 0.8696902534150501], "top": "permitted", "option_logits": null, "prompt_sha256": "6e590e2135227d5cca28d2e60e24fd54e93c992a155bfc23179ee4c580915112", "input_tokens": 211, "server_ms": 33.95}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88200004398823, "probs": [0.048789556914816484, 0.020270905643033604, 0.93093953744215], "top": "permitted", "option_logits": null, "prompt_sha256": "3f39f09c7d9e21ffc1f78839202c597a69077efd392d449211c28f002576f82d", "input_tokens": 202, "server_ms": 34.16}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.170291878283024, "probs": [0.21264458466138933, 0.7457413364088059, 0.04161407892980468], "top": "insufficient", "option_logits": null, "prompt_sha256": "faac46392097109effab43e6f29e0f0866a09da218a45f79f5778c2e5a65a73d", "input_tokens": 204, "server_ms": 34.23}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67260706424713, "probs": [0.058304340167377615, 0.02006813943899786, 0.9216275203936245], "top": "permitted", "option_logits": null, "prompt_sha256": "dbea7157a3c37b6742682cae5b072b1054dd3ad36476b8a23466297be6d5f638", "input_tokens": 214, "server_ms": 33.75}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31310414150357, "probs": [0.08617432324783239, 0.8780222847407805, 0.03580339201138707], "top": "prohibited", "option_logits": null, "prompt_sha256": "de7b2b0f8c3b1dea2ae34d68854d8932985f8194c5d9ef94ef9ac3c5af3abf22", "input_tokens": 214, "server_ms": 34.53}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98346018791199, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "1aff6b562ec348487995092a4cf68d351f0a76eea07f42c0842730b8f60281f4", "input_tokens": 213, "server_ms": 34.02}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41961601190269, "probs": [0.13537010767547805, 0.41875814284681956, 0.4458717494777024], "top": "insufficient", "option_logits": null, "prompt_sha256": "a0ea731df0f9f147afafd6706401757982b609f60c79d64c03ebfea463ef9821", "input_tokens": 212, "server_ms": 34.48}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.038890179246664, "probs": [0.06461138798541208, 0.9008867824895763, 0.034501829525011594], "top": "prohibited", "option_logits": null, "prompt_sha256": "dd3955a315dde371dc5e4ce9792fa9390e46cc33b9d2c6592ffb03bce46f1738", "input_tokens": 216, "server_ms": 34.36}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44617593474686, "probs": [0.043034915318676176, 0.9309127048683882, 0.026052379812935716], "top": "permitted", "option_logits": null, "prompt_sha256": "ef8d68233fa0d3bb7526e3c22aa69a27e6f6fbda1b973de57cbe90e38379d4c0", "input_tokens": 216, "server_ms": 34.61}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11736109107733, "probs": [0.08884445347431193, 0.060974765390393336, 0.8501807811352947], "top": "permitted", "option_logits": null, "prompt_sha256": "72458905854c2bd8458210955b878067b9d46d1b7595a462d19f98f25c4fbfa2", "input_tokens": 208, "server_ms": 34.28}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62193815782666, "probs": [0.21015003684890857, 0.05285696229504507, 0.7369930008560464], "top": "insufficient", "option_logits": null, "prompt_sha256": "1eb890e1e9003931d9ddcfe01bc6929ca77211b8a8ab9237c546de8c3001faa3", "input_tokens": 215, "server_ms": 34.88}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04494214616716, "probs": [0.11741505362023608, 0.2826034863954738, 0.5999814599842902], "top": "permitted", "option_logits": null, "prompt_sha256": "5167011ba9be4d3d5e4675e3946f7775214985b6acc67629d42da678b376b719", "input_tokens": 209, "server_ms": 34.18}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41793497465551, "probs": [0.42551441767130554, 0.06085086611336375, 0.5136347162153307], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ae6464839efd58c0371c609fe15542b4468085854cec4634bb01da9bb661271", "input_tokens": 212, "server_ms": 34.6}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07573190331459, "probs": [0.29549446261685514, 0.11530520829956423, 0.5892003290835807], "top": "prohibited", "option_logits": null, "prompt_sha256": "9231010e808cb51e90e91e319911586ed3aeb9b5c6d6c648c044df8b8287b77e", "input_tokens": 209, "server_ms": 34.38}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.525607040151954, "probs": [0.5169331333226727, 0.45597494786051085, 0.02709191881681649], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6546fb5bf3590e9fba39e6d78cfd36e6155363cfafeb691260ad754365b88a2", "input_tokens": 210, "server_ms": 34.66}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11646213196218, "probs": [0.028170092258100337, 0.026457059155058287, 0.9453728485868413], "top": "A", "option_logits": null, "prompt_sha256": "2c3c3a83be0ee81ef6d6592eccaec7e6a6ba5b2ea0d6391d9d86d5e775337c59", "input_tokens": 200, "server_ms": 34.25}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27061301469803, "probs": [0.032411181205817395, 0.9594367541920572, 0.008152064602125513], "top": "B", "option_logits": null, "prompt_sha256": "91903b178a53be97c6b4e21b1d0650ef3e043300c347ae23ae561a14234e3d7e", "input_tokens": 201, "server_ms": 34.55}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03732090070844, "probs": [0.9231307043235732, 0.06218066343701687, 0.014688632239409829], "top": "B", "option_logits": null, "prompt_sha256": "3dff5c64bbce09d87dfb74fabb20e7031a8984b18f71249b72bfd902a583b649", "input_tokens": 192, "server_ms": 34.19}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5064869876951, "probs": [0.04181158538259194, 0.05373823788322395, 0.904450176734184], "top": "A", "option_logits": null, "prompt_sha256": "cf4e23a2812a6cdaf0dfd19302a5eda22c6153968a2782074212b2be313cfc91", "input_tokens": 192, "server_ms": 34.7}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48694511502981, "probs": [0.9579542284336563, 0.039062801461222305, 0.0029829701051215123], "top": "B", "option_logits": null, "prompt_sha256": "29a2165d171c923e1807421722f38341ede9d1fb3e93a0c5373c0d3ba55ae49a", "input_tokens": 208, "server_ms": 33.8}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67230694554746, "probs": [0.9780535330693921, 0.014616188125699304, 0.007330278804908576], "top": "A", "option_logits": null, "prompt_sha256": "21515e5ecf0e10fc73cbe4c24cbbd88c4224cae289a96d2b0e594ae3597b31e8", "input_tokens": 211, "server_ms": 33.9}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62065602838993, "probs": [0.9618961177282845, 0.02691947679068596, 0.011184405481029442], "top": "A", "option_logits": null, "prompt_sha256": "6ce513c832609614a5c86a1da793e3dc9479da4bd0d08df92ef7858289130727", "input_tokens": 200, "server_ms": 33.83}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.699727177619934, "probs": [0.17805852075193587, 0.15706137389923117, 0.664880105348833], "top": "A", "option_logits": null, "prompt_sha256": "a496e0a84ecb3fab0bcbcbf67c42d43175ca71241f413bb499c2d25cf5b89415", "input_tokens": 190, "server_ms": 33.88}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.542054964229465, "probs": [0.07003258404452323, 0.012871804091497973, 0.9170956118639788], "top": "A", "option_logits": null, "prompt_sha256": "601893abb138ce73b2b6942d4c60e2602f97eb9895db7c93790ee0080d24b941", "input_tokens": 203, "server_ms": 33.77}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78682795725763, "probs": [0.03871909899223181, 0.9495254842123682, 0.011755416795400006], "top": "B", "option_logits": null, "prompt_sha256": "d3cf5c157fb76c1b82e4fd00b70aad0c853e81926e95dc24dca56c08cb0094d2", "input_tokens": 206, "server_ms": 34.01}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31297398917377, "probs": [0.028505129122276228, 0.07305053773408933, 0.8984443331436344], "top": "B", "option_logits": null, "prompt_sha256": "cc8fa7a50f3be03bcd0afd14f9ec758a2e5a1b0b7ef163beb146ecabb71b14ee", "input_tokens": 197, "server_ms": 34.45}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.558077136054635, "probs": [0.7064924652629315, 0.0651228142271108, 0.22838472050995764], "top": "insufficient", "option_logits": null, "prompt_sha256": "8726e9f6e7ede98264291ff8ad0e45430c6ce3426738746ae7ec330c5225073c", "input_tokens": 192, "server_ms": 34.68}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.106061121448874, "probs": [0.9439935183065499, 0.0409858846475417, 0.0150205970459084], "top": "B", "option_logits": null, "prompt_sha256": "bbdddbef606c62e9ac9a597096a51234b94035a57674ce04db983e275d076a86", "input_tokens": 203, "server_ms": 34.37}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40586503408849, "probs": [0.06642565405281799, 0.9261833743555155, 0.0073909715916665505], "top": "A", "option_logits": null, "prompt_sha256": "f9062a341f3150d9232fc2091c0f4a720c9f8a7a9ebf2a3b53f7945cc81b561f", "input_tokens": 203, "server_ms": 34.55}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20318200066686, "probs": [0.9772298659128125, 0.012098405641787949, 0.010671728445399525], "top": "A", "option_logits": null, "prompt_sha256": "3efd8eb9f9fa0e03f2533097d8073cf8265b904bdaf5e12d7e71cbcda034ebae", "input_tokens": 193, "server_ms": 34.48}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54860698245466, "probs": [0.09931756389846377, 0.8926071429310916, 0.008075293170444685], "top": "insufficient", "option_logits": null, "prompt_sha256": "931bb93c879b95601b770f5bcc531d57aca3c51f1e201a92f41a4af0d559e6c5", "input_tokens": 192, "server_ms": 34.68}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.944710183888674, "probs": [0.06639387877077424, 0.007865755914174356, 0.9257403653150514], "top": "A", "option_logits": null, "prompt_sha256": "5d4a9ae116daa6161534b5519583cd012b52060548dce72ac7d0f781876c0ce5", "input_tokens": 208, "server_ms": 33.95}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.877819104120135, "probs": [0.025425185481267775, 0.9673249441344611, 0.007249870384270957], "top": "B", "option_logits": null, "prompt_sha256": "6a579e3ca68fa3d28ee162c2bb9061af2372a8a728baca705895776608d58850", "input_tokens": 206, "server_ms": 34.06}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62109700962901, "probs": [0.900473990544217, 0.08837786536850668, 0.01114814408727632], "top": "B", "option_logits": null, "prompt_sha256": "98330242bf9b4853692144bd273028a1065e63fecbd536b99f72041a124cbdde", "input_tokens": 200, "server_ms": 33.74}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13673306629062, "probs": [0.1414487087141659, 0.4960584710750265, 0.3624928202108077], "top": "A", "option_logits": null, "prompt_sha256": "86f3509930c216c6bc17217666f92763bc1fa437854da9186a9a14e952bf3b17", "input_tokens": 193, "server_ms": 34.19}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53005510568619, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 33.7}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68841696344316, "probs": [0.05584400041528186, 0.9398915591084774, 0.004264440476240678], "top": "A", "option_logits": null, "prompt_sha256": "104d6ac93e38d37c2cbd2fe1d413e52ca92dde1258cac35fc220528b73196b21", "input_tokens": 209, "server_ms": 33.85}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27357415482402, "probs": [0.04937458683512937, 0.008523069447562875, 0.9421023437173077], "top": "A", "option_logits": null, "prompt_sha256": "d217d38b7e6213bb691099e21c66a3df1e10fe6cde75e36ed398dd45fd11b8e9", "input_tokens": 194, "server_ms": 34.47}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.485486129298806, "probs": [0.38826805862544134, 0.5657334002422568, 0.04599854113230185], "top": "insufficient", "option_logits": null, "prompt_sha256": "47a14c3c4a4be0db70f00c92b4b884aa58fae59d2d69bfaa15a6dd001c74977e", "input_tokens": 192, "server_ms": 34.6}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.40476311556995, "probs": [0.05525329298111996, 0.014797133073951734, 0.9299495739449284], "top": "contradicted", "option_logits": null, "prompt_sha256": "bb827fab3f7af6904a86bfc213ce42d7a2cf4557d68a9ad04f7b2711c1308a09", "input_tokens": 216, "server_ms": 33.64}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.353045074269176, "probs": [0.8170038444540815, 0.05859532327955457, 0.12440083226636395], "top": "supported", "option_logits": null, "prompt_sha256": "17fe8aeab450890957725a1dad43750752505b560e3976c468dc3f41172f8a67", "input_tokens": 214, "server_ms": 34.59}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60182608291507, "probs": [0.9735218387884688, 0.00998474638199737, 0.01649341482953388], "top": "supported", "option_logits": null, "prompt_sha256": "238376c4cc5fbc7cd76f9ad9cb9e509ce751a7fd9c3b9a11286be50239d869ec", "input_tokens": 209, "server_ms": 33.76}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44199592620134, "probs": [0.9553471931059949, 0.01618550073428256, 0.02846730615972255], "top": "insufficient", "option_logits": null, "prompt_sha256": "ce06953b256550886167647265795ea3869e5ff8f5316771e55ce1518c69ddd5", "input_tokens": 207, "server_ms": 34.49}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64263593964279, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 33.79}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7025570012629, "probs": [0.4209957860789075, 0.3076410529313528, 0.27136316098973967], "top": "permitted", "option_logits": null, "prompt_sha256": "d51944708b071233f3970b04f6c6336f8e70e3809df2ba49e0aabdd866ad8bdc", "input_tokens": 229, "server_ms": 33.81}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6856069304049, "probs": [0.05387521829671438, 0.906755673529099, 0.03936910817418649], "top": "permitted", "option_logits": null, "prompt_sha256": "8bff8d1de44bca4e15fd82d70b8f339fee2bd109ec9a92c9747b89a54e021644", "input_tokens": 222, "server_ms": 34.01}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90223884768784, "probs": [0.9459403411775168, 0.04372962453450628, 0.010330034287976873], "top": "insufficient", "option_logits": null, "prompt_sha256": "00c4b02298270b667458c96645eee6cce2724fb4365acaf92619d1772f6df80f", "input_tokens": 221, "server_ms": 34.05}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.426373990252614, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.64}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76383709535003, "probs": [0.9863441380986024, 0.010771259663552918, 0.00288460223784459], "top": "A", "option_logits": null, "prompt_sha256": "ff213732225a439e059b46a57708fb2a00c963c13243e933f189700505c04cf5", "input_tokens": 210, "server_ms": 34.0}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.398153053596616, "probs": [0.14639175402671709, 0.7964845624457851, 0.05712368352749774], "top": "A", "option_logits": null, "prompt_sha256": "336f079f69772f717f7a6c3f445805134d5e53ef7b61fb0ad8ebcf52b418a553", "input_tokens": 209, "server_ms": 33.64}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71489795669913, "probs": [0.9283260069346223, 0.02597998847580144, 0.045694004589576176], "top": "insufficient", "option_logits": null, "prompt_sha256": "37ba551c43c8c232aa20632adb709ac0b29072c10a739d0094e4f45e5095146d", "input_tokens": 211, "server_ms": 33.98}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71717597171664, "probs": [0.03832591832238971, 0.9398833253029221, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "f87a6aa4ea4be148073907bfbd331ed2bc23ab8a4dd102101ddfab9df68affa9", "input_tokens": 226, "server_ms": 33.86}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65225696749985, "probs": [0.15261824181393646, 0.7798668927662274, 0.06751486541983612], "top": "supported", "option_logits": null, "prompt_sha256": "414bff4be61247980ece53db3773f05e7b7026c988db7d1528e992a308174423", "input_tokens": 227, "server_ms": 33.91}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.42191388644278, "probs": [0.02224167487575335, 0.959332479722117, 0.01842584540212969], "top": "supported", "option_logits": null, "prompt_sha256": "6f4011c71525c9b091a8816574bd328fde25b6ba7ab7b96ded01b3a730a3d931", "input_tokens": 202, "server_ms": 33.73}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99131007306278, "probs": [0.05252409837620808, 0.9412533141801623, 0.006222587443629545], "top": "insufficient", "option_logits": null, "prompt_sha256": "5c9d452b9464173f19e40416da519c21dd6ab53b7cde7ba01038e5e77b34ea1b", "input_tokens": 202, "server_ms": 34.2}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.533704958856106, "probs": [0.08806019173320215, 0.12050723322647901, 0.7914325750403188], "top": "prohibited", "option_logits": null, "prompt_sha256": "a8173efc87a69fb87bd0cca2d2a6bd8745dc3278ebad157ac546bb3f41987882", "input_tokens": 233, "server_ms": 33.72}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97588015347719, "probs": [0.5880203043739818, 0.24430850840437723, 0.16767118722164093], "top": "permitted", "option_logits": null, "prompt_sha256": "8aecf205f28a5179380e1ecfd6e459d887b2402409823f5046b6caa992897f9a", "input_tokens": 231, "server_ms": 34.09}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.27619193866849, "probs": [0.8133507124046239, 0.05478605406972841, 0.13186323352564766], "top": "permitted", "option_logits": null, "prompt_sha256": "bcca8c96798eb11f3f4644bd6b48d9956d0ef797acb354a9d313e72d7decc3e6", "input_tokens": 216, "server_ms": 33.56}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.974329967051744, "probs": [0.15147579954487175, 0.20728921703153022, 0.6412349834235981], "top": "insufficient", "option_logits": null, "prompt_sha256": "9109d62aa99372ff918463de6ec1fd8982d3a2a6d2840a8c9977e79bd2ab935b", "input_tokens": 223, "server_ms": 34.22}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92761994898319, "probs": [0.9535382502516264, 0.03221192217085971, 0.014249827577513809], "top": "A", "option_logits": null, "prompt_sha256": "a10dad7ab11ae42542c0d0b776d3f586dc16a28cb00984bccb5c89dc77075c2f", "input_tokens": 222, "server_ms": 34.03}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.811788100749254, "probs": [0.04901004980173283, 0.9351466866030752, 0.015843263595192024], "top": "B", "option_logits": null, "prompt_sha256": "76d3d59ab8383e39c3b6c2101903ac065a99bf3d49d00c16e61ce375f0f7bc7c", "input_tokens": 222, "server_ms": 33.94}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.544925067573786, "probs": [0.8768319468998427, 0.09162951349252214, 0.03153853960763513], "top": "B", "option_logits": null, "prompt_sha256": "83a38a50ea29d93de3d3e3463c307552985b65ef7fbecfb33f19f0c8b19cbe78", "input_tokens": 215, "server_ms": 33.84}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03731088899076, "probs": [0.04595900357971523, 0.02033122661038582, 0.933709769809899], "top": "insufficient", "option_logits": null, "prompt_sha256": "5483a1f853a7dd5c96bd7c5c666a4e199aff64cda6caf94dcccb07b14eae9bfc", "input_tokens": 201, "server_ms": 34.28}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.398012889549136, "probs": [0.04052399739483789, 0.026120737232413562, 0.9333552653727485], "top": "contradicted", "option_logits": null, "prompt_sha256": "dd4cc5375267c593d2a03b475e5c365a6578823acd2a704447d9b76168008824", "input_tokens": 223, "server_ms": 33.66}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76251810975373, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "supported", "option_logits": null, "prompt_sha256": "d4d2d98ff82872fd16a73e2e6edb11f4ecab9360d684a9727d45783d301f374f", "input_tokens": 219, "server_ms": 33.89}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.890999879688025, "probs": [0.026724635554949777, 0.9549340005022864, 0.018341363942763936], "top": "supported", "option_logits": null, "prompt_sha256": "296f774f1cfc7e7122448c98bb6e3a52dc18135394acb94f5473c12727592101", "input_tokens": 220, "server_ms": 33.93}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70664597302675, "probs": [0.14195326239124903, 0.8223426360308026, 0.03570410157794831], "top": "insufficient", "option_logits": null, "prompt_sha256": "9b5808bbbac1566fd1810d826dd09771c70684f8e1c3c1505e9a93a728995747", "input_tokens": 218, "server_ms": 34.04}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.885048961266875, "probs": [0.0674490002480741, 0.883262901735513, 0.04928809801641285], "top": "prohibited", "option_logits": null, "prompt_sha256": "dfbe932d725d8fa600a07771813057adf2f6237700a213c275d2418103feffd5", "input_tokens": 224, "server_ms": 34.14}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89663903787732, "probs": [0.17474597379856985, 0.4768198017317031, 0.34843422446972716], "top": "prohibited", "option_logits": null, "prompt_sha256": "faf8ebe7a328f98f41892d79a870c982c804cd410a13ccd121f1e747020314b3", "input_tokens": 221, "server_ms": 34.09}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.46704414859414, "probs": [0.049073786622505555, 0.0715038981480174, 0.879422315229477], "top": "permitted", "option_logits": null, "prompt_sha256": "cd350bbed40603bfdec8aee19ed8f4aead08aae6c3e75e5950990320c9a1602f", "input_tokens": 221, "server_ms": 33.63}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02926100231707, "probs": [0.039087581768329945, 0.9002715143788037, 0.06064090385286648], "top": "insufficient", "option_logits": null, "prompt_sha256": "32a2ba65d112e44959b5167505e017413199fda0540cd2a995538aa795dabd11", "input_tokens": 220, "server_ms": 34.28}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.46293398924172, "probs": [0.019768802923085988, 0.013567511661018838, 0.9666636854158951], "top": "A", "option_logits": null, "prompt_sha256": "6c3a458921faa1b3dc850b93cbb1d6ed2a1453f181ed0aeb0df4c5ae9cccf625", "input_tokens": 227, "server_ms": 33.73}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67773609049618, "probs": [0.02394587818769938, 0.9700312518040514, 0.006022870008249252], "top": "B", "option_logits": null, "prompt_sha256": "0d78a5511291c0e8891da715a82ec2ad438e94f5dddaee4ff0a75c528e00ad7e", "input_tokens": 230, "server_ms": 33.92}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75133688375354, "probs": [0.028667490838461873, 0.9620652951327192, 0.00926721402881876], "top": "B", "option_logits": null, "prompt_sha256": "f925b35d49f8ca5aac7190490beee3fe1f89e7ea2183f46b6ff8de5291cc99d6", "input_tokens": 223, "server_ms": 33.96}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82789904996753, "probs": [0.022241675019567426, 0.01842584578673998, 0.9593324791936926], "top": "insufficient", "option_logits": null, "prompt_sha256": "836107065b1093b68346fcce5e1adc9aba25fe8c243775524b77da14167cf552", "input_tokens": 210, "server_ms": 34.08}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66821587830782, "probs": [0.055084494859207855, 0.9271085820941714, 0.017806923046620855], "top": "contradicted", "option_logits": null, "prompt_sha256": "a88acfe95a51b8a25895a91b153a497406a07078525e2b97bc3c403e06a540b7", "input_tokens": 227, "server_ms": 33.98}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88744897954166, "probs": [0.05252720299692694, 0.8840676728179807, 0.0634051241850924], "top": "supported", "option_logits": null, "prompt_sha256": "60d8999e12533dd71716a6a7531ccfc7ec7f2073d709301bdd11b0ac9658b776", "input_tokens": 224, "server_ms": 34.09}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52645484544337, "probs": [0.10359906505840241, 0.12505351454311125, 0.7713474203984862], "top": "insufficient", "option_logits": null, "prompt_sha256": "673717072f8a9221f1adf01785a87c22cc50bf85f7bb720009c52de94dae71e4", "input_tokens": 220, "server_ms": 33.65}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63279807753861, "probs": [0.04112297491157286, 0.011726020323995183, 0.9471510047644319], "top": "insufficient", "option_logits": null, "prompt_sha256": "63e2228bb7d5b362f1e995b7d1a0c9d791b2ec99854b33a504865fb77a469bad", "input_tokens": 208, "server_ms": 34.8}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99211007915437, "probs": [0.2873316476131822, 0.23803638895290566, 0.4746319634339121], "top": "permitted", "option_logits": null, "prompt_sha256": "bf284395337716d2923af2416dde0e555c17bde0e0c5ea382e5010db93daadea", "input_tokens": 227, "server_ms": 34.23}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74678807146847, "probs": [0.18941262861483832, 0.8018310660162423, 0.008756305368919429], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ba65298c9d27b7445ffc33c5502792b0a58a03c427a8f2b68f062202870179", "input_tokens": 228, "server_ms": 33.98}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13789186440408, "probs": [0.045449474437266706, 0.031192392974493464, 0.9233581325882398], "top": "permitted", "option_logits": null, "prompt_sha256": "81df99f7f22c5619613f932264037a494407bd0c9c1cd772d88cdb13bd93f038", "input_tokens": 225, "server_ms": 34.25}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53046691417694, "probs": [0.06610715660723597, 0.9217425227678987, 0.012150320624865412], "top": "insufficient", "option_logits": null, "prompt_sha256": "720a483b782512dcd38b2b0081ceb423a67c83842981d937ef45d7a508910f4c", "input_tokens": 218, "server_ms": 34.75}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68956714682281, "probs": [0.4315972806039035, 0.5547093438603172, 0.013693375535779267], "top": "B", "option_logits": null, "prompt_sha256": "4e84221098fdfcea2eadf56d4d09744125fba5bd9394ccda4463841127ab114b", "input_tokens": 221, "server_ms": 33.76}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3468438629061, "probs": [0.8253003742309912, 0.1256641025406766, 0.04903552322833224], "top": "A", "option_logits": null, "prompt_sha256": "bd5f90a9191f0f7973e6d324cf6a49506e2cd54e3cfc5de85edbfda099804d5f", "input_tokens": 227, "server_ms": 34.49}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00040117837489, "probs": [0.18006428080502881, 0.8116115753175274, 0.008324143877443896], "top": "A", "option_logits": null, "prompt_sha256": "e536eabe724f1b5913aff9c542223b70d01472d2b42b51b49d37b52e912739a5", "input_tokens": 207, "server_ms": 34.24}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.919189151376486, "probs": [0.0642990095007749, 0.09368810379830601, 0.842012886700919], "top": "insufficient", "option_logits": null, "prompt_sha256": "d1260818c5ac8e709aa2420a7f131bcdacd6f95c3ee49deeed13973d0a110795", "input_tokens": 197, "server_ms": 34.16}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67860594578087, "probs": [0.9649135185430301, 0.019733010925224156, 0.015353470531745737], "top": "contradicted", "option_logits": null, "prompt_sha256": "d3371673a5e78756742dbd1370ab4b5bf6fb4c9e37f1bf9f6b2f8fc296fa2b3d", "input_tokens": 213, "server_ms": 33.86}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56068607233465, "probs": [0.025219845781526033, 0.015267532821545986, 0.959512621396928], "top": "supported", "option_logits": null, "prompt_sha256": "11f0f8368eb8d7e7794ac745fa45e0bc9dc4b1ac9ed01cd9acc0b7783d4aaf6b", "input_tokens": 212, "server_ms": 33.82}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70043800957501, "probs": [0.9604600229721326, 0.012660690572914277, 0.026879286454953178], "top": "supported", "option_logits": null, "prompt_sha256": "3af4ed7c8a0907629d9650ab957ccf4b7c6a7dda4e2a6dd2995d5bb1743e52e3", "input_tokens": 202, "server_ms": 34.01}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.190282018855214, "probs": [0.027209446310095767, 0.8576061514878427, 0.11518440220206158], "top": "contradicted", "option_logits": null, "prompt_sha256": "c0ec0e2875ae2fe16b5af804b0c765df404b9b4f5d893843240547ff3251f572", "input_tokens": 203, "server_ms": 34.43}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.52507602237165, "probs": [0.8574932428756057, 0.12262617681966377, 0.01988058030473045], "top": "prohibited", "option_logits": null, "prompt_sha256": "5d48756022f5bf582cc4dcaecef3a33be36788da1e16cde33b69afb8e41d6d0c", "input_tokens": 218, "server_ms": 33.66}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4482951592654, "probs": [0.036422612390539545, 0.9510408592905502, 0.012536528318910257], "top": "permitted", "option_logits": null, "prompt_sha256": "1e8882942aa8fcb63c49bda41019547d5c388d1eb722d8f3d2f6ff0fe2f6de83", "input_tokens": 221, "server_ms": 34.59}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.26652201637626, "probs": [0.03036360301441411, 0.9570210386636204, 0.012615358321965421], "top": "permitted", "option_logits": null, "prompt_sha256": "58d1fda856fe23c325b5eb40b4f3316df39757b9358721f866eaee369d56311c", "input_tokens": 215, "server_ms": 33.55}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74053703434765, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "7e3b871d1d0377bf269820f9b1f1d027e38c15d4fc3b1ad841733cfa7eaa1caa", "input_tokens": 220, "server_ms": 33.99}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53165581636131, "probs": [0.9856903364066905, 0.012203148876708984, 0.0021065147166005127], "top": "A", "option_logits": null, "prompt_sha256": "49ab3244841a47c40baa28769654a2caa6beec9c9be7b8902fa30d48d3df3ae9", "input_tokens": 206, "server_ms": 33.67}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.788278026506305, "probs": [0.026052379812935716, 0.043034915318676176, 0.9309127048683882], "top": "B", "option_logits": null, "prompt_sha256": "3e92d9a3ec71acb771522ae129c59a1c3f3cf135471b1a9119034e8e60f57534", "input_tokens": 212, "server_ms": 33.93}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81242885068059, "probs": [0.9773173707852877, 0.018771280462410337, 0.003911348752301854], "top": "B", "option_logits": null, "prompt_sha256": "c8b102c19db70a1095b4a0e3f5b0fc428173a9dc8eeba42de46dbbde084b4a9e", "input_tokens": 203, "server_ms": 34.04}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64472606033087, "probs": [0.03778914105307011, 0.035491168622697584, 0.9267196903242324], "top": "insufficient", "option_logits": null, "prompt_sha256": "4a91f5a0c83e6f4ce998b7f04fc311297f577c57be12b145cf649e0aa08f1703", "input_tokens": 191, "server_ms": 33.96}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.37790307402611, "probs": [0.7336632938165211, 0.04359087300806054, 0.2227458331754184], "top": "contradicted", "option_logits": null, "prompt_sha256": "71d93a27e5ea2ee304e786d87ef894fb7755249d7fb8613e14d533e8fd133800", "input_tokens": 224, "server_ms": 33.68}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96221997961402, "probs": [0.06296298480307037, 0.877902829429632, 0.05913418576729761], "top": "supported", "option_logits": null, "prompt_sha256": "ffb22e022a23a832611c31cce83e5ded9325f32d02e4447965afe9987da2bde6", "input_tokens": 221, "server_ms": 34.08}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.695427959784865, "probs": [0.04392538756600811, 0.9501750287751061, 0.005899583658885871], "top": "supported", "option_logits": null, "prompt_sha256": "6075f3ad5119128a82986b4f0679052bba33e8403d8fb22ec4c8f7ed10d93946", "input_tokens": 229, "server_ms": 33.72}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72616695985198, "probs": [0.9704787445693202, 0.01201482490914338, 0.01750643052153659], "top": "insufficient", "option_logits": null, "prompt_sha256": "48b9296dd657bda90e5f3b2c63f69eebef357ae1a9850de04e5cfd329a5ceeba", "input_tokens": 226, "server_ms": 33.85}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2357030864805, "probs": [0.3304014836417537, 0.5811151968205441, 0.08848331953770218], "top": "prohibited", "option_logits": null, "prompt_sha256": "f2f72ef9b5a8872981b046c25a1b9e9511405cfacc72a16d99f362fea882a19c", "input_tokens": 226, "server_ms": 34.42}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.605217192322016, "probs": [0.11071640116478274, 0.011569936207086172, 0.877713662628131], "top": "permitted", "option_logits": null, "prompt_sha256": "13277d62009fb53093658090b337dad57eca4375e0abe61be5ee35f2ec7e4157", "input_tokens": 215, "server_ms": 34.74}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.293313071131706, "probs": [0.17087454466914134, 0.3005367666543428, 0.5285886886765159], "top": "permitted", "option_logits": null, "prompt_sha256": "cecc81469d7fbe04a5075ff2cb74c748f517953000b249e9f63c274d01b5b1d9", "input_tokens": 225, "server_ms": 34.58}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57866704836488, "probs": [0.5363636986202766, 0.44434397563891254, 0.019292325740810894], "top": "insufficient", "option_logits": null, "prompt_sha256": "072c7c6a806afdb07685f8b1f3c43852c2e6b32f1c5516b65df03793dc2bb53f", "input_tokens": 224, "server_ms": 34.81}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22985298372805, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.41}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53628581762314, "probs": [0.657302153396938, 0.15527126967431085, 0.1874265769287511], "top": "B", "option_logits": null, "prompt_sha256": "3b58b4916fb300ada6ef30438bc6d52ac081154f25e9849d672afc231c6278ff", "input_tokens": 218, "server_ms": 34.77}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34096419066191, "probs": [0.019992665974153154, 0.06184594114611621, 0.9181613928797306], "top": "B", "option_logits": null, "prompt_sha256": "d02d406417fcec2c9d2cbd7690bc752f474c89ea8c64602b69bbcceffa6d243a", "input_tokens": 209, "server_ms": 34.53}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51634596846998, "probs": [0.08689224299716879, 0.08160829342180988, 0.8314994635810213], "top": "insufficient", "option_logits": null, "prompt_sha256": "af78451b7d18177d0037af51b7830934b233d57812e77dd145bc82ebf8f94400", "input_tokens": 207, "server_ms": 34.78}], "seconds": 5.3, "failures": 0}, "negative": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.464415887370706, "probs": [0.06566437538338366, 0.8598927385077062, 0.07444288610891006], "top": "insufficient", "option_logits": null, "prompt_sha256": "2111f6636d10660f740e0f032908924d0640e60a712060af19c6e8972d95e5c9", "input_tokens": 204, "server_ms": 34.72}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.611967185512185, "probs": [0.2945662267563794, 0.03956295740237862, 0.665870815841242], "top": "contradicted", "option_logits": null, "prompt_sha256": "ed5aa8d62ce8778e40b8c51f1d1fe90b1f392ab252247908adc4e84b2546ab08", "input_tokens": 202, "server_ms": 34.75}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.31452417559922, "probs": [0.9241406947470572, 0.051569175242271295, 0.024290130010671554], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6f696700120ee981b3e3e0dc237f67896db84867803273e124be96bfd72fecc", "input_tokens": 196, "server_ms": 34.57}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.74372797831893, "probs": [0.11215130215053613, 0.052825544241278395, 0.8350231536081855], "top": "contradicted", "option_logits": null, "prompt_sha256": "ef4b71a8ae0a1bc3711450f77606ddabd2be62c9c2b6587e1fcacbccfcb9cb00", "input_tokens": 196, "server_ms": 33.89}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.162331864237785, "probs": [0.15307015041158983, 0.014109646797217717, 0.8328202027911924], "top": "insufficient", "option_logits": null, "prompt_sha256": "215840810bc2483aa303cd79f5d4f92093a49aa9957dfa697f20989aa1390d55", "input_tokens": 209, "server_ms": 34.32}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.38508396781981, "probs": [0.03329916574926079, 0.8166100641505288, 0.1500907701002104], "top": "supported", "option_logits": null, "prompt_sha256": "fc0d6607f95e1b53fe2cfa889a2cf1bfed2a95e2da0424c85742050fd8b5cde7", "input_tokens": 206, "server_ms": 34.61}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.28885389864445, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "supported", "option_logits": null, "prompt_sha256": "f3bf4ddaa8f4178e0b5f59c09fffe0c07bfd4bd34d616bbf9577361257ca1afa", "input_tokens": 208, "server_ms": 34.42}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.487796042114496, "probs": [0.3541639151292647, 0.5160413123689802, 0.1297947725017551], "top": "contradicted", "option_logits": null, "prompt_sha256": "e5578ff7fa77b4e9f8593ede9b8e31ecccdca126f02444f3c94cdd0bb01ee6ca", "input_tokens": 200, "server_ms": 34.71}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.36907499842346, "probs": [0.017835936327419456, 0.8721495360638406, 0.11001452760873995], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b2ae5222808313b49d4b84a395fe3c5b48345949fa231e2c8fba100ed470a94", "input_tokens": 194, "server_ms": 34.54}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.123851012438536, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b42ed76cec9fa4a467694c0069d312883b296f2ee2385059235881dee283be", "input_tokens": 194, "server_ms": 34.19}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.632077069953084, "probs": [0.03186734083364, 0.0247947114952375, 0.9433379476711226], "top": "contradicted", "option_logits": null, "prompt_sha256": "d2c60f4c3e6615ad87fdb4cd474622ea0a7f3f3b3fd7df582b5539d02a9dbd1b", "input_tokens": 193, "server_ms": 33.86}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.849648924544454, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "contradicted", "option_logits": null, "prompt_sha256": "89f547d1f4dbfccb3c2e3443f18b4576344bd16efeaec8108d2febb0c913ec9e", "input_tokens": 192, "server_ms": 34.04}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.118551786988974, "probs": [0.03540454160362492, 0.924457767599498, 0.04013769079687701], "top": "contradicted", "option_logits": null, "prompt_sha256": "6beeb2f79fa6330dc68f2b769cf2779997d4b57c459bf40113bbc11a80353072", "input_tokens": 205, "server_ms": 34.22}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.05310099758208, "probs": [0.028539740573724572, 0.7452090392508762, 0.22625122017539928], "top": "insufficient", "option_logits": null, "prompt_sha256": "b094a32aab8f6f92e09b2405572effe6bec478d6d3b5985cdeb71dab794ed591", "input_tokens": 205, "server_ms": 34.07}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.4439741242677, "probs": [0.026796810283021767, 0.844600378585805, 0.12860281113117333], "top": "insufficient", "option_logits": null, "prompt_sha256": "4e199755434441474e22e920ad5f8587827e30212d91945271ceadbb26337008", "input_tokens": 199, "server_ms": 33.62}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.05121111124754, "probs": [0.1303439116251751, 0.013621023271142042, 0.8560350651036829], "top": "contradicted", "option_logits": null, "prompt_sha256": "d75aa0e009db26e5e38e3bfd553573eccf95a0526360da23bada050f23e908f8", "input_tokens": 194, "server_ms": 34.2}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.79559798911214, "probs": [0.1369403800499024, 0.018392355065480374, 0.8446672648846173], "top": "supported", "option_logits": null, "prompt_sha256": "9e994209df771b9f8829dd54c36907e41ed241a507cce230c7e3505302a9fa1e", "input_tokens": 198, "server_ms": 33.99}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.734648048877716, "probs": [0.7651520472632017, 0.10276696393112311, 0.13208098880567515], "top": "contradicted", "option_logits": null, "prompt_sha256": "45785396c292fe5cbe62d99a45714d49c993d6f5b5783769cec1795f7b84f35f", "input_tokens": 198, "server_ms": 33.85}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.579495994374156, "probs": [0.9249067812606918, 0.05495367443647756, 0.020139544302830666], "top": "contradicted", "option_logits": null, "prompt_sha256": "490eb806b0921b46aa816c18534ea40c0719e89eb76c34feb4e89d55c921bf36", "input_tokens": 194, "server_ms": 33.74}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.15658211149275, "probs": [0.0190993007707109, 0.7736996936098133, 0.2072010056194758], "top": "contradicted", "option_logits": null, "prompt_sha256": "7e29e327fc2aef321d690607a7c7347871c7eb3dd07ac6d45327aab094603d91", "input_tokens": 194, "server_ms": 34.2}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.70840710401535, "probs": [0.021206655711830842, 0.5200601388697108, 0.45873320541845836], "top": "insufficient", "option_logits": null, "prompt_sha256": "ffb27e8dfc1d8298a601b7f483c42104d68f6e8ebfea086896afd140756b8c6c", "input_tokens": 211, "server_ms": 33.8}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.55450597964227, "probs": [0.9097361205627902, 0.05076536678270011, 0.039498512654509606], "top": "supported", "option_logits": null, "prompt_sha256": "e7855c00f037441fcdc5ff3fefaa61ddaeb0e79e09ba6c0c6043c767ee6d98ca", "input_tokens": 211, "server_ms": 34.72}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.24108403548598, "probs": [0.9223434223370909, 0.04004589025644397, 0.03761068740646525], "top": "supported", "option_logits": null, "prompt_sha256": "1a0c7ecb563009263a7969e56c6e00fc34f99b8ad08b4db579ed9e2bc78f4c6f", "input_tokens": 201, "server_ms": 34.48}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.52661682665348, "probs": [0.1051237595843464, 0.007539451491827663, 0.8873367889238258], "top": "contradicted", "option_logits": null, "prompt_sha256": "610dae3d86ea66ae02b343c18c1478ef66be2619fe3a6c03ff05a434fac307d6", "input_tokens": 197, "server_ms": 34.67}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.10158099420369, "probs": [0.7409240241997488, 0.18635729477403776, 0.07271868102621346], "top": "insufficient", "option_logits": null, "prompt_sha256": "52c15cca48120ff357156a28827735982f0407b36b9806dc79a94193e52ed2a4", "input_tokens": 212, "server_ms": 34.37}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.157151848077774, "probs": [0.9070425995126343, 0.07852475778895956, 0.01443264269840612], "top": "prohibited", "option_logits": null, "prompt_sha256": "5df6c8de75514db23d6e53210ba8920e11a83a08e31ffd76867eb6d789db3f29", "input_tokens": 218, "server_ms": 34.38}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.31893398799002, "probs": [0.04968326826324694, 0.890344493113758, 0.05997223862299509], "top": "insufficient", "option_logits": null, "prompt_sha256": "3dc91e8f64358377c40956ef40870819732d6e66fc8ed44530ee8b7b1e32f1e6", "input_tokens": 203, "server_ms": 34.45}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.94341005757451, "probs": [0.09374299764974467, 0.009200493913293934, 0.8970565084369614], "top": "prohibited", "option_logits": null, "prompt_sha256": "866701419f4e54bef33f2151b0e4c0c830728dbfd94c6ba4a3fbd92b438151a2", "input_tokens": 209, "server_ms": 34.17}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.11023204959929, "probs": [0.20575303419639995, 0.025954061735184695, 0.7682929040684153], "top": "prohibited", "option_logits": null, "prompt_sha256": "6c14eaaad2ab386d458a6458fce2b79bcd81ecbfc3c66679c950d808dd0c2a3f", "input_tokens": 209, "server_ms": 34.29}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.5742269679904, "probs": [0.8315971903507323, 0.10489918128594154, 0.06350362836332618], "top": "permitted", "option_logits": null, "prompt_sha256": "d8b705bc82e40643b8dcd135b43f67a8e3671ecc2f28e81b48de6aa1cbb84792", "input_tokens": 214, "server_ms": 34.75}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.09030104801059, "probs": [0.258643657502053, 0.6628301324407543, 0.07852621005719278], "top": "insufficient", "option_logits": null, "prompt_sha256": "5000e3b05641e28823047872c8bd19f4604d76e0fa38c23bdad3d8e7be7bce61", "input_tokens": 209, "server_ms": 34.4}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.543727084994316, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "prohibited", "option_logits": null, "prompt_sha256": "97ddaad2b5074dab6f31a5a7a8ded12c3e51123aa5d819f19c006ba123fde089", "input_tokens": 209, "server_ms": 34.77}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.73516702838242, "probs": [0.09658208517581993, 0.868022296339126, 0.03539561848505407], "top": "insufficient", "option_logits": null, "prompt_sha256": "5783bf781c97c761780f8d8454e12c8895251116d4d5ebe264401d9d9dcae0a1", "input_tokens": 208, "server_ms": 33.92}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.75813809968531, "probs": [0.0651099549445377, 0.02705164492548852, 0.9078384001299739], "top": "prohibited", "option_logits": null, "prompt_sha256": "a92081d320caa09114cab155b336249985672a93e86328503ed4397aa61c3293", "input_tokens": 211, "server_ms": 33.93}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.753398133441806, "probs": [0.049132112383896635, 0.729412594713007, 0.22145529290309635], "top": "insufficient", "option_logits": null, "prompt_sha256": "2b0f13909e189564a5e5099e9094600d4804ae6ef7ef59e7435f85302977a0c0", "input_tokens": 202, "server_ms": 33.96}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.25684410892427, "probs": [0.1638594512396548, 0.04974899459992729, 0.7863915541604178], "top": "prohibited", "option_logits": null, "prompt_sha256": "a4ffa08e79c520845fb80c116412fe4c48304f5f003b514bbcbea0a7c61cfe7b", "input_tokens": 204, "server_ms": 34.35}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.53209516778588, "probs": [0.06162034871506134, 0.8591822512326693, 0.07919740005226927], "top": "insufficient", "option_logits": null, "prompt_sha256": "f51b6f4e075e67f76ff574d496ba2f9c19cc2ca38a2f56d989791df67251d344", "input_tokens": 214, "server_ms": 33.68}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.70079680159688, "probs": [0.7621344563262837, 0.23138990123980285, 0.006475642433913465], "top": "insufficient", "option_logits": null, "prompt_sha256": "baf1674ea5ff54f508700c815711c06e9a50808663840042c147d883c72021ed", "input_tokens": 214, "server_ms": 33.92}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.56500484794378, "probs": [0.14398191537109406, 0.021923360098142518, 0.8340947245307634], "top": "insufficient", "option_logits": null, "prompt_sha256": "8854be3865426f818cb7189838d63dc61b9b64da67d13087c8e6ccc2a708a8c0", "input_tokens": 213, "server_ms": 33.81}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.9948809966445, "probs": [0.2129289153471605, 0.545676239155193, 0.24139484549764645], "top": "permitted", "option_logits": null, "prompt_sha256": "913468ed99926586c9292a6dabcee9b083a7b1b1c719b6bad31fda7f7f045a2e", "input_tokens": 212, "server_ms": 34.28}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.80243901908398, "probs": [0.15943768499721103, 0.025848589203191098, 0.814713725799598], "top": "insufficient", "option_logits": null, "prompt_sha256": "acfb724b1ffcab3d0820c0a1f4cdb189396280c1faf4e622ce1f306cad1bf45e", "input_tokens": 216, "server_ms": 33.99}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.72113688662648, "probs": [0.9471510047644319, 0.04112297491157286, 0.011726020323995183], "top": "insufficient", "option_logits": null, "prompt_sha256": "7a9df7814cddba377022c022111d998db3d8a2ff48c109eb7f1aac0f5c151319", "input_tokens": 216, "server_ms": 34.0}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.572626093402505, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "ecd27e22648cba485966b925a1b12fd5a888331b66634b0f8919798224cf8072", "input_tokens": 208, "server_ms": 33.81}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.22200309857726, "probs": [0.2729138208779564, 0.3294318709588002, 0.3976543081632435], "top": "insufficient", "option_logits": null, "prompt_sha256": "86759c52a7cc961194391587ca2df8d3998c59ec236637e4acc083f151ad5242", "input_tokens": 215, "server_ms": 34.24}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.80705791339278, "probs": [0.5472741025611921, 0.331307576591957, 0.12141832084685092], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ee81091d38ca686274af8a9057ae962583e07c84d4a7c8913246f20a9adc153", "input_tokens": 209, "server_ms": 33.84}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.79991792887449, "probs": [0.10569064817366915, 0.056437743537509265, 0.8378716082888217], "top": "insufficient", "option_logits": null, "prompt_sha256": "d890e328d92ed0294fd0ca0f45891d715f1b53fae8d48848ee13c936721cc9f2", "input_tokens": 212, "server_ms": 33.93}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.73225804232061, "probs": [0.745864682612535, 0.0779433517872937, 0.17619196560017134], "top": "permitted", "option_logits": null, "prompt_sha256": "866975030f9845962316982b48126dfbaa42cd89b8d3719305c5c1cd2a7ce35c", "input_tokens": 209, "server_ms": 33.77}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.999770905822515, "probs": [0.30671712087126574, 0.5394587819774863, 0.153824097151248], "top": "permitted", "option_logits": null, "prompt_sha256": "33ed49b156b02a8251a71c89906619f22d3cff122755a441bc701809f452ec03", "input_tokens": 210, "server_ms": 34.07}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.826418014243245, "probs": [0.9411304198850982, 0.03837677130689544, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "26a5967a0d4f86a5e648fdf739a14ae8463a48c7dc0319e4b9026afcfc70ce49", "input_tokens": 200, "server_ms": 33.91}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.77049791440368, "probs": [0.2191677303480843, 0.5616645393038313, 0.2191677303480843], "top": "insufficient", "option_logits": null, "prompt_sha256": "5cb564ef00f9a81eb96ac41fe01780b4b9ffdcb60d7bc8217b9d89860c279ba3", "input_tokens": 201, "server_ms": 33.93}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.25692194327712, "probs": [0.8246471790493917, 0.16138164146161363, 0.013971179488994609], "top": "insufficient", "option_logits": null, "prompt_sha256": "e747a01e40c9029fd1004867b0db224405c02f421c64f74cd748920300cb2000", "input_tokens": 192, "server_ms": 33.54}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.910389084368944, "probs": [0.9136677967042812, 0.07428832300095413, 0.012043880294764588], "top": "B", "option_logits": null, "prompt_sha256": "3d5649d07c298093a047c4d77967454e627c1fd4792f41873c4640c036a163cd", "input_tokens": 192, "server_ms": 34.13}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.727318074554205, "probs": [0.807231387784515, 0.17909249095841714, 0.013676121257067941], "top": "insufficient", "option_logits": null, "prompt_sha256": "88ac81fbf23c9684447ca373863f9cf4c54ca0bd35eb3e960680db4187456d96", "input_tokens": 208, "server_ms": 33.81}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.76508809439838, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "insufficient", "option_logits": null, "prompt_sha256": "7263692449eeef173103012af0a80bd27db0a89dc35bf50d4fed73b80a29f045", "input_tokens": 211, "server_ms": 33.93}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.60218603909016, "probs": [0.046119873005046885, 0.9369780144263187, 0.016902112568634386], "top": "B", "option_logits": null, "prompt_sha256": "4c2c0840c1ad7e181d30f10eb62a4618c3195e936034bc7274fe3c8fc945d786", "input_tokens": 200, "server_ms": 33.87}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.359524983912706, "probs": [0.6742384798146065, 0.29826788550402233, 0.02749363468137107], "top": "B", "option_logits": null, "prompt_sha256": "2d0f22653ae25b1b9ad253802809222e0554c7bd42292a5a1637a04128e5c52f", "input_tokens": 190, "server_ms": 34.58}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.16327203437686, "probs": [0.8763745016908776, 0.09751141040687612, 0.026114087902246324], "top": "insufficient", "option_logits": null, "prompt_sha256": "676f3a162c1d11c22fd5f07e5b57e6b6e6d3a4aae515a7c2965aeac124b616dd", "input_tokens": 203, "server_ms": 34.33}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.328673992305994, "probs": [0.13968031972950484, 0.23073250176366764, 0.6295871785068274], "top": "insufficient", "option_logits": null, "prompt_sha256": "109526bdb944267ab212d1b8ba2f77a3a6c7c147016cf89ae5bc2e0e8e661adc", "input_tokens": 206, "server_ms": 34.61}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.702587036415935, "probs": [0.9673249441344611, 0.025425185481267775, 0.007249870384270957], "top": "insufficient", "option_logits": null, "prompt_sha256": "c4889b691603599fd8dca223a645f5e88b7248a364a9bef64c37a8a9e3ced5eb", "input_tokens": 197, "server_ms": 33.89}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.87566006556153, "probs": [0.49248819366063284, 0.16951261805505752, 0.3379991882843097], "top": "insufficient", "option_logits": null, "prompt_sha256": "5deb35c130c31229df54964339f9faccffa08f3ed2eb38730d86ac9ccd510aa3", "input_tokens": 192, "server_ms": 34.18}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.61818499676883, "probs": [0.1558415388693792, 0.330859461294187, 0.5132989998364338], "top": "insufficient", "option_logits": null, "prompt_sha256": "7c27c7c3361bd04c584a975218ab6b610dcab0f9bf6078cf3870b470410c383c", "input_tokens": 203, "server_ms": 33.78}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.7579679004848, "probs": [0.30165258046080173, 0.13344430167683133, 0.5649031178623669], "top": "B", "option_logits": null, "prompt_sha256": "27dba9d4c38e8b61907893c325f5851e4726ba89fa7ac9e5563a14abe89239b9", "input_tokens": 203, "server_ms": 33.97}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.673466907814145, "probs": [0.9480355945752582, 0.038658346366163006, 0.013306059058578745], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f3dd4d0a864d976d5c35a7b088b3aed4b55500043acc8701dd4d35729125db8", "input_tokens": 193, "server_ms": 33.79}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.944359075278044, "probs": [0.22389674812152066, 0.7374540645868362, 0.03864918729164314], "top": "A", "option_logits": null, "prompt_sha256": "164c8915823587dee72ef8800598282728e580a02992fe7a8d8fa557ead4e3b9", "input_tokens": 192, "server_ms": 34.09}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.57870600000024, "probs": [0.9379420530362722, 0.04616732447007806, 0.01589062249364978], "top": "insufficient", "option_logits": null, "prompt_sha256": "d7c03673e95db552913947e72213d55f0938d187ca0d4fd69182cbe141bfda65", "input_tokens": 208, "server_ms": 33.77}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.00005984865129, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "A", "option_logits": null, "prompt_sha256": "b54ee64662a21acf112153e17512f10436c1938df35ab9ad182be1ff0c47c3e0", "input_tokens": 206, "server_ms": 34.17}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.611835937947035, "probs": [0.13579878113602123, 0.8376256951826758, 0.02657552368130303], "top": "A", "option_logits": null, "prompt_sha256": "8aa1b7f158bf57460e6f9f057913e0d01d913f16998cc0a46327e8aa6c93e838", "input_tokens": 200, "server_ms": 33.81}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.18466288410127, "probs": [0.25821814633948337, 0.25821814633948337, 0.4835637073210332], "top": "A", "option_logits": null, "prompt_sha256": "b17c06e9ca0e91accd4cfcb4667c5c3a42687d3209e1ecd98910eb1343a18dc2", "input_tokens": 193, "server_ms": 34.12}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.55287506990135, "probs": [0.12660014777476133, 0.09251256875552054, 0.7808872834697181], "top": "insufficient", "option_logits": null, "prompt_sha256": "324bb2046ede92322b8175d6af2961ad208eba5f52712692465ef944de2193a4", "input_tokens": 206, "server_ms": 33.77}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.85296908952296, "probs": [0.9299397393693557, 0.06263931303365462, 0.007420947596989686], "top": "insufficient", "option_logits": null, "prompt_sha256": "86e7d75bb9b41e2d9de272239185c5cce5c19dc1aac6573a737c31d47ab0563f", "input_tokens": 209, "server_ms": 33.91}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.3478748369962, "probs": [0.37290349783252347, 0.01111172675798736, 0.6159847754094891], "top": "insufficient", "option_logits": null, "prompt_sha256": "a1a07677dc816ad01403f9cff2867d5bfdaff2c29d8a4c2293ce0662a24f33ae", "input_tokens": 194, "server_ms": 34.47}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.94386011920869, "probs": [0.475785940079552, 0.1976777825029813, 0.3265362774174668], "top": "insufficient", "option_logits": null, "prompt_sha256": "82b8973ae0b97456180d256ef08822a11d20365d8039148ed863e9bb75c9bda4", "input_tokens": 192, "server_ms": 34.03}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.717056995257735, "probs": [0.14585095569321507, 0.009226876843769601, 0.8449221674630154], "top": "insufficient", "option_logits": null, "prompt_sha256": "9a672222518ab31da9c3ab2072849ef03b511e8e9d23762f89d33a83feca6ec1", "input_tokens": 216, "server_ms": 33.7}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.945179803296924, "probs": [0.09629593898139954, 0.7633942252849403, 0.14030983573366013], "top": "contradicted", "option_logits": null, "prompt_sha256": "5d2348206184a53d2da714668c8004522fc660c04abc09a4675170278b6d0ce8", "input_tokens": 214, "server_ms": 34.03}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.67906695045531, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "497e689287006b51bd4f178ebc9023050c338fca8abf790eef2112975de7f243", "input_tokens": 209, "server_ms": 33.73}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.8379179853946, "probs": [0.9369780144263187, 0.046119873005046885, 0.016902112568634386], "top": "supported", "option_logits": null, "prompt_sha256": "984a11157ce42cd37ec6209d96b53bf0d293025d6b69161f1a358f9bcc9d57db", "input_tokens": 207, "server_ms": 33.97}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.990250926464796, "probs": [0.6452505747343799, 0.2680863307491953, 0.08666309451642484], "top": "permitted", "option_logits": null, "prompt_sha256": "ea5a71f7bf051b93ca522c5ff3caef6ffdfcb5011f24af51a404b10524b250cb", "input_tokens": 228, "server_ms": 33.97}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.08435106091201, "probs": [0.43020843462499875, 0.1903144997976119, 0.3794770655773894], "top": "permitted", "option_logits": null, "prompt_sha256": "b5ac43b527f17cc117db3df096bff98320c5a248422c7b513eedeba99e619731", "input_tokens": 229, "server_ms": 34.15}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.61143593490124, "probs": [0.056437743537509265, 0.8378716082888217, 0.10569064817366915], "top": "insufficient", "option_logits": null, "prompt_sha256": "be77461c7c49b38137b436f74bcc533cf9fdcc1fbbcabe3acbe8c8c37f2272bd", "input_tokens": 222, "server_ms": 33.79}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.94862010888755, "probs": [0.1110623891201585, 0.008481107191351438, 0.88045650368849], "top": "permitted", "option_logits": null, "prompt_sha256": "12c28c53461194e05d3490b99aecb157c1965b942f5bfee5ef8163543d2bb1d0", "input_tokens": 221, "server_ms": 33.98}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.65802604518831, "probs": [0.7598853370160624, 0.23070704972212444, 0.009407613261813141], "top": "insufficient", "option_logits": null, "prompt_sha256": "0b7bc0317c30767061abcd7c39f9b71942a3c49b230cf334d1e6db7873b4dc61", "input_tokens": 216, "server_ms": 33.67}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.02871990390122, "probs": [0.9718671022270775, 0.019875215614779472, 0.008257682158143088], "top": "B", "option_logits": null, "prompt_sha256": "282f482d7bf4f65f4cc0f8dd6ca05d597f32f80783d49b010173ef79b130ffc5", "input_tokens": 210, "server_ms": 34.14}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.56285488978028, "probs": [0.08284327606718382, 0.0730741626697933, 0.8440825612630228], "top": "A", "option_logits": null, "prompt_sha256": "6bf8f97ede01d029938070b85d9e3aa545cbbbb84007fb406100f0a0a8ffa474", "input_tokens": 209, "server_ms": 33.59}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.99030098505318, "probs": [0.1753450388599366, 0.7903403209860906, 0.03431464015397285], "top": "A", "option_logits": null, "prompt_sha256": "33dc6c8523ece68e76db4eaffbb8a8470964368747977448381041503e3ad36c", "input_tokens": 211, "server_ms": 33.97}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.066251039505005, "probs": [0.03629282833642113, 0.01605512937357001, 0.9476520422900088], "top": "supported", "option_logits": null, "prompt_sha256": "6e4467088790119e033569f3066af7eedf2f72c934657e3a20082f685de045f2", "input_tokens": 226, "server_ms": 34.02}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.24635299295187, "probs": [0.044616056760944024, 0.6623675120954924, 0.2930164311435636], "top": "insufficient", "option_logits": null, "prompt_sha256": "1a718aab5e212abb4604d4e57ad3d7fb8cba756be66eba7d483eec463cb18c6b", "input_tokens": 227, "server_ms": 34.31}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.578275030478835, "probs": [0.04013769079687701, 0.03540454160362492, 0.924457767599498], "top": "contradicted", "option_logits": null, "prompt_sha256": "bafd2feb34ff693920168afa57cc5bc050eea256c82f2e9e071467805a7857f4", "input_tokens": 202, "server_ms": 33.7}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.46206592768431, "probs": [0.21057073282474073, 0.0031468025509083547, 0.7862824646243509], "top": "supported", "option_logits": null, "prompt_sha256": "c4c6394f10ee2df26d3dac75e02000ae56f28719ec24b10add7af6e77f5a4889", "input_tokens": 202, "server_ms": 34.35}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.83047811500728, "probs": [0.10502328955939726, 0.2865713245281423, 0.6084053859124604], "top": "permitted", "option_logits": null, "prompt_sha256": "37bd9a494e79ba1e91bbf2a68704609c66421b3ec11ab8a5bdd36908b17135b1", "input_tokens": 233, "server_ms": 33.96}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.113501923158765, "probs": [0.2680863307491953, 0.08666309451642484, 0.6452505747343799], "top": "insufficient", "option_logits": null, "prompt_sha256": "a115c9f1aaee5b00fa1fd3be52a599afa7112fdcf9eec8871ae98b3d6ef2183f", "input_tokens": 231, "server_ms": 34.13}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.836238112300634, "probs": [0.07184163899867897, 0.8298456430479688, 0.09831271795335216], "top": "prohibited", "option_logits": null, "prompt_sha256": "100e5f7a276a412d51e2ddf940777a1ff109ea7803a8291d638760c392064d80", "input_tokens": 216, "server_ms": 33.85}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.53056703135371, "probs": [0.7469915530638082, 0.14618461002062086, 0.106823836915571], "top": "prohibited", "option_logits": null, "prompt_sha256": "6486b8dc13e88f4d5e489f3b3b76c6bb5aeed63f19005132e10380f0181726aa", "input_tokens": 223, "server_ms": 34.57}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.7652780842036, "probs": [0.8638851317960057, 0.051328051461926705, 0.08478681674206763], "top": "B", "option_logits": null, "prompt_sha256": "941cd3d853797aa1a92c8261e9339af72809062c590b26ef2db8bdbbc5d68191", "input_tokens": 222, "server_ms": 33.83}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.63788798823953, "probs": [0.046253998446132885, 0.9397029314342114, 0.014043070119655722], "top": "A", "option_logits": null, "prompt_sha256": "f9c6ae0da8d46bec4df2b4102f3aebb45748add11af2d63a981d879fea6a717c", "input_tokens": 222, "server_ms": 34.59}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.17342182062566, "probs": [0.04067015270682889, 0.7760155022030047, 0.1833143450901663], "top": "insufficient", "option_logits": null, "prompt_sha256": "4df509501a3fa1e95f377f6b2cd7ff2737ebe52dd6e95472a1a35d0fc43e3fdf", "input_tokens": 215, "server_ms": 34.21}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.307454040274024, "probs": [0.05543534738188995, 0.9330136575005004, 0.01155099511760962], "top": "A", "option_logits": null, "prompt_sha256": "d68329bad204407dddf0266c2d55c897537804b87978786e2c29649197957a18", "input_tokens": 201, "server_ms": 34.29}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.7548877838999, "probs": [0.06878691906304032, 0.03042978117418096, 0.9007832997627787], "top": "supported", "option_logits": null, "prompt_sha256": "3fdb716c314c67c93413daa3cd5a38b13bec265f706d90a97c43e5a108104f13", "input_tokens": 223, "server_ms": 33.81}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.18298301100731, "probs": [0.06893683236028562, 0.08321304657583695, 0.8478501210638775], "top": "contradicted", "option_logits": null, "prompt_sha256": "d68195061969b189d71df192da0191d4d56877cf1d8ce0d4479bc8898c998c94", "input_tokens": 219, "server_ms": 34.35}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.724746925756335, "probs": [0.016115540131028547, 0.8390475816237557, 0.14483687824521582], "top": "insufficient", "option_logits": null, "prompt_sha256": "6edd6bca63afcb6fb6d263e3e248e24e556fde2b01d965155729342ef37e411e", "input_tokens": 220, "server_ms": 33.75}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.112242076545954, "probs": [0.2862501142494634, 0.02478133063598072, 0.6889685551145559], "top": "contradicted", "option_logits": null, "prompt_sha256": "d113705714c5c456d527c8466aeda96099d32a8069c13bf0f40b1ff3ed43ad11", "input_tokens": 218, "server_ms": 34.14}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.33051405288279, "probs": [0.1609098519153101, 0.7722359149508133, 0.06685423313387646], "top": "insufficient", "option_logits": null, "prompt_sha256": "19fd16742b6e54b9e2452e09592de10d5fc44c96cab1630e5f7651b1af77d898", "input_tokens": 224, "server_ms": 34.34}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.92699014209211, "probs": [0.3043910694191852, 0.39121786116162954, 0.3043910694191852], "top": "insufficient", "option_logits": null, "prompt_sha256": "ea512df6fc353426e37889a53cae2118b9e832157a4af067cb4a85489a5c83ff", "input_tokens": 221, "server_ms": 33.9}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.55064501799643, "probs": [0.08097077759111387, 0.04060828666999563, 0.8784209357388906], "top": "prohibited", "option_logits": null, "prompt_sha256": "71ad1455eaea177dccc5d9fc4078b19378e2ce1c1d27289d9120a1c445349487", "input_tokens": 221, "server_ms": 33.58}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.72815805859864, "probs": [0.8979988326296015, 0.06048782007179681, 0.041513347298601795], "top": "permitted", "option_logits": null, "prompt_sha256": "31939a3c71e268cd4751d57ccd6a07ff44f1ca1f81c7c2f7bf165ea50668fe0a", "input_tokens": 220, "server_ms": 34.69}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.755637964233756, "probs": [0.9821934763214742, 0.008345422472445314, 0.009461101206080565], "top": "B", "option_logits": null, "prompt_sha256": "13f4ee2ce11c253ccf784222a18f900b57830384e577f4adf0030d9893ed01a9", "input_tokens": 227, "server_ms": 33.88}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.0912819635123, "probs": [0.9584447268648851, 0.039082802533792566, 0.0024724706013222815], "top": "insufficient", "option_logits": null, "prompt_sha256": "c8a6bf0c918df98324f40d3c989bf2d3593c8114ebaba9510a0de7a63d90c43f", "input_tokens": 230, "server_ms": 33.9}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.71324695460498, "probs": [0.8152011530118691, 0.1808606650321593, 0.003938181955971634], "top": "insufficient", "option_logits": null, "prompt_sha256": "d512a04940165a8dc92bcf5e8a905fa84148aca8078cba93d34897ca73ebe5fc", "input_tokens": 223, "server_ms": 33.71}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.89243900589645, "probs": [0.9509567466006987, 0.028336479497072638, 0.020706773902228682], "top": "A", "option_logits": null, "prompt_sha256": "726c7f274cbd758a0855ced1b8e06e30b0888e289eeb459d087a868fb9610ef5", "input_tokens": 210, "server_ms": 34.04}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.7037879768759, "probs": [0.03787087428859695, 0.033405034339472804, 0.9287240913719302], "top": "supported", "option_logits": null, "prompt_sha256": "0032df826a7e49e7be6e1c8d5f96ece8822a858eeb6f5280c55a79fe2e628bbb", "input_tokens": 227, "server_ms": 33.8}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.34727390110493, "probs": [0.04458832792770586, 0.7990410839770169, 0.15637058809527726], "top": "insufficient", "option_logits": null, "prompt_sha256": "c29cf273a3d78467e10369969a81a42df358b37cb972ad44f78d455f8785b7e8", "input_tokens": 224, "server_ms": 34.26}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.797638051211834, "probs": [0.6079293570442296, 0.046423581791889615, 0.34564706116388083], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b9e9fd277a7252eb95c0dea4201d1d75394e8dd03ee09f576f6fc0355881f3", "input_tokens": 220, "server_ms": 33.75}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.07139194011688, "probs": [0.07421350543321283, 0.7101726109045401, 0.2156138836622472], "top": "contradicted", "option_logits": null, "prompt_sha256": "25fc9b80402930a472ddf5fbb9527f22c905747ffefc59e4865da0bcf422e5bb", "input_tokens": 208, "server_ms": 34.13}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.90465912222862, "probs": [0.3596689227288795, 0.5240624893078483, 0.11626858796327226], "top": "prohibited", "option_logits": null, "prompt_sha256": "1a9834724cc432424395c6e3e4989cae7698e00f85fedac3d9a8cd1a698cd265", "input_tokens": 227, "server_ms": 33.82}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.19788300804794, "probs": [0.17244943152985545, 0.1836151222987057, 0.6439354461714388], "top": "permitted", "option_logits": null, "prompt_sha256": "cc59abf9dfd3346910e8d93dc78fe4cd21d0c20962370b801e609b82712c7ea7", "input_tokens": 228, "server_ms": 34.18}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.05085115507245, "probs": [0.11048645425133781, 0.06688603847853027, 0.8226275072701319], "top": "insufficient", "option_logits": null, "prompt_sha256": "45e75124dce70ee83cbbd2df43d7c5f3045b0da83559f8b4772e8cb6fa82dbb0", "input_tokens": 225, "server_ms": 34.27}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.03046101145446, "probs": [0.07600853330506088, 0.8779775900702179, 0.046013876624721124], "top": "permitted", "option_logits": null, "prompt_sha256": "902f5bf4dd52878d854ff660af610beb5cd00553cb6cf9fa5c1e7a5ea010391b", "input_tokens": 218, "server_ms": 34.15}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.78441792726517, "probs": [0.07014840077352269, 0.16883849753435962, 0.7610131016921177], "top": "B", "option_logits": null, "prompt_sha256": "d4a4f3c22ce6befbff78058a364e4617bda379057c95b5bacf2410a162ea27f7", "input_tokens": 221, "server_ms": 33.98}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.933918949216604, "probs": [0.7195969840802272, 0.2637194956142592, 0.016683520305513583], "top": "insufficient", "option_logits": null, "prompt_sha256": "fc0371a2d78013d514b134511d38b37de6e71a39ad2fa9b95b0119945db7a316", "input_tokens": 227, "server_ms": 34.2}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.40842414461076, "probs": [0.6328238128438389, 0.029254643239392513, 0.33792154391676854], "top": "A", "option_logits": null, "prompt_sha256": "5521ebbd2304bdf6049132465a2b8c7b28c862f462f50f98d90efeb483601e64", "input_tokens": 207, "server_ms": 33.66}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.086121970787644, "probs": [0.22351946247571317, 0.12708523693891682, 0.64939530058537], "top": "A", "option_logits": null, "prompt_sha256": "c11ad01ccd57e2b3b849c07b89887124412bce4a8e5506adaeb954f23c598ac7", "input_tokens": 197, "server_ms": 34.34}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.23703301511705, "probs": [0.038658346366163006, 0.013306059058578745, 0.9480355945752582], "top": "supported", "option_logits": null, "prompt_sha256": "8d5e868c2065409c5dcedc10c1d875c102dfbb48d2433b234f0ad0456f1049bc", "input_tokens": 213, "server_ms": 34.18}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.72350710257888, "probs": [0.01953747345276732, 0.9553520377742281, 0.025110488773004678], "top": "contradicted", "option_logits": null, "prompt_sha256": "6ac7738979b2b31bef2f7e2f3d8984b0b2b559ee71fa4d2b728e28884e3183b5", "input_tokens": 212, "server_ms": 33.81}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.843299167230725, "probs": [0.05100292527189731, 0.035003776957469146, 0.9139932977706335], "top": "contradicted", "option_logits": null, "prompt_sha256": "66c4b4c8b3dc52b623b51acf50a373bf625ad5559da7c3bf90c47f56e7c2829b", "input_tokens": 202, "server_ms": 33.95}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.43556491099298, "probs": [0.09842495451008554, 0.01699017293024806, 0.8845848725596664], "top": "insufficient", "option_logits": null, "prompt_sha256": "7db376c4d8c96d2c69fda861429911890ec2d70b0f375e60f65c846f1693e18a", "input_tokens": 203, "server_ms": 34.34}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.86785907484591, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "03449c2d41ee64f00d22b4b6e619aa69ce6d623d00b4972f503f5ed336bba08e", "input_tokens": 218, "server_ms": 33.79}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.80645814165473, "probs": [0.9331701885712249, 0.04890646327936172, 0.017923348149413328], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ed1d2ff423bdb9e622336508299a3180255754a84d1847b9b5ba5d1d5623692", "input_tokens": 221, "server_ms": 33.9}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.209952948614955, "probs": [0.05755147392994642, 0.0327217252711575, 0.9097268007988961], "top": "insufficient", "option_logits": null, "prompt_sha256": "fb0073f867c77a04f9287adbd88276517a2a85e5cf9dcfb04c81ff5ba768e03f", "input_tokens": 215, "server_ms": 34.29}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.24038298241794, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "insufficient", "option_logits": null, "prompt_sha256": "51c4ac799c31ceb752046516252cb8d59c3a3c1654c685c82476530a4d74a689", "input_tokens": 220, "server_ms": 34.24}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.93761001341045, "probs": [0.040967527214491276, 0.8323022347202245, 0.12673023806528433], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a5a9b08e7ca06d1f5808cdcd3270c79b064cd4fed16b4be278fc7d5f58a1d1f", "input_tokens": 206, "server_ms": 34.0}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.238702876493335, "probs": [0.048847636440916636, 0.07578275550521898, 0.8753696080538644], "top": "insufficient", "option_logits": null, "prompt_sha256": "555fd1e019506f048448ae23a6bac4fe7f3e9f2fe60181be02acfdf8225318e5", "input_tokens": 212, "server_ms": 34.35}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.970739951357245, "probs": [0.07319171247064139, 0.2264135372721482, 0.7003947502572104], "top": "insufficient", "option_logits": null, "prompt_sha256": "1fc713f5dfc86b0d90f75094b087d920fbfeb9830fb4eec74f78d2f1d67fbf66", "input_tokens": 203, "server_ms": 33.93}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.195992888882756, "probs": [0.9387496456243781, 0.055776151759141294, 0.005474202616480596], "top": "B", "option_logits": null, "prompt_sha256": "77e6633b56fd111f3cf38ff30d1c308c084b3de608efb9682cf02409dc31c9db", "input_tokens": 191, "server_ms": 34.32}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.89179918728769, "probs": [0.46903312718147117, 0.38856479455009935, 0.14240207826842943], "top": "supported", "option_logits": null, "prompt_sha256": "83104242d8900e7e51e6ca6f99d29588121dd7153c0fda9d487e3eafef91196e", "input_tokens": 224, "server_ms": 33.96}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.891089053824544, "probs": [0.7720397533404616, 0.11755438599864407, 0.11040586066089443], "top": "insufficient", "option_logits": null, "prompt_sha256": "66f0b52c1eee120f16069ed75fbd7bfc556b5a3dc09c27f809799b32ae74e959", "input_tokens": 221, "server_ms": 34.0}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.76301799714565, "probs": [0.014236196905258059, 0.6140385266223682, 0.3717252764723737], "top": "insufficient", "option_logits": null, "prompt_sha256": "fac76f29223ea6573fb84bb8cb70a27292765e7f4f90ebb19001e8a0b7bf0533", "input_tokens": 229, "server_ms": 33.88}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.96604003198445, "probs": [0.06291220787817095, 0.0030966605414949147, 0.9339911315803341], "top": "supported", "option_logits": null, "prompt_sha256": "89df7654ae643d961d5bd0a1cf472717b674fe14a8f5377a6918c1103303e14f", "input_tokens": 226, "server_ms": 34.04}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.53157502412796, "probs": [0.5216500126916775, 0.26161676918070537, 0.21673321812761712], "top": "prohibited", "option_logits": null, "prompt_sha256": "73a309dd6a1605e1d928e2a4947516f3ae54fda513d77cea9d4fdf892dbbebb5", "input_tokens": 226, "server_ms": 33.54}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.205782951787114, "probs": [0.843310768581531, 0.1455727910071266, 0.01111644041134243], "top": "prohibited", "option_logits": null, "prompt_sha256": "97510bceb98f70589de7351d3f810c2e4c175b03bbf6ddec55eca00ef9c199ee", "input_tokens": 215, "server_ms": 34.2}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.19575307331979, "probs": [0.42929779219364844, 0.486689574582582, 0.08401263322376958], "top": "permitted", "option_logits": null, "prompt_sha256": "5a1d1ca84275c476cd245cafcd0e1fd36598a6dae3c643b8ea9d60c5b57d6535", "input_tokens": 225, "server_ms": 34.38}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.020351039245725, "probs": [0.19560247361171754, 0.23611000151056785, 0.5682875248777147], "top": "permitted", "option_logits": null, "prompt_sha256": "fec6fd7a8c54ee7856b56b8cc85aa3ff4cb617820fd52f7faf204e2efb4a86ff", "input_tokens": 224, "server_ms": 34.07}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.68969706632197, "probs": [0.9323898466010938, 0.02450695094155103, 0.04310320245735524], "top": "B", "option_logits": null, "prompt_sha256": "fcc6cca56205c973fe5ee42de7bbca225406234b5346f63e55af7d8c5f115d9a", "input_tokens": 213, "server_ms": 33.68}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.20373194664717, "probs": [0.643833389199536, 0.3438005390976864, 0.012366071702777563], "top": "insufficient", "option_logits": null, "prompt_sha256": "176f3f577d5802518ba6bb8a36039f8c2c7fa4991aac2e236870e0189d0b31a6", "input_tokens": 218, "server_ms": 34.19}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 38.60264900140464, "probs": [0.020042268594912593, 0.21743089555343206, 0.7625268358516554], "top": "B", "option_logits": null, "prompt_sha256": "8494413c37184f76cf07859eef5fe11f7373b1acb0da7cbd005226a5bb00610d", "input_tokens": 209, "server_ms": 35.27}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 39.47813808917999, "probs": [0.33058297459294167, 0.050336110784327995, 0.6190809146227304], "top": "A", "option_logits": null, "prompt_sha256": "70f353d85ca31feb6cd6d5c421fe7a2e877a313657d44902068dc2f6ef451bf8", "input_tokens": 207, "server_ms": 36.27}], "seconds": 5.4, "failures": 0}, "multifield": {"rows": [{"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.065505059435964, "probs": [0.14720923592665844, 0.8527907640733415], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.065505059435964, "probs": [0.8214121479792841, 0.17115665028167817, 0.00743120173903778], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.065505059435964, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.065505059435964, "probs": [0.9305404571279347, 0.06673814075012655, 0.0027214021219388934], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.66331796720624, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.66331796720624, "probs": [0.8095965925134007, 0.07462670936879237, 0.11577669811780701], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.66331796720624, "probs": [0.9471470189565372, 0.052852981043462664], "top": "same"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.66331796720624, "probs": [0.9350414422025257, 0.05915293776423124, 0.005805620033243134], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.44739478640258, "probs": [0.22187782400103293, 0.778122175998967], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.44739478640258, "probs": [0.09874410354776683, 0.06776898998409085, 0.8334869064681424], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.44739478640258, "probs": [0.07967467169628852, 0.9203253283037116], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.44739478640258, "probs": [0.10841793826059654, 0.0843556864709907, 0.8072263752684127], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.14869302511215, "probs": [0.21123545993246395, 0.788764540067536], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.14869302511215, "probs": [0.11796301878836102, 0.8782943442297925, 0.0037426369818465326], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.14869302511215, "probs": [0.8527907640733415, 0.14720923592665844], "top": "same"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.14869302511215, "probs": [0.24350964936246988, 0.7532804299591004, 0.0032099206784296486], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.25421287305653, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.25421287305653, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.25421287305653, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.25421287305653, "probs": [0.9344875966425453, 0.06294564908924384, 0.002566754268210852], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.5395668502897, "probs": [0.7318060856600669, 0.2681939143399331], "top": "moved"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.5395668502897, "probs": [0.727805131100519, 0.25050783114702185, 0.02168703775245924], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.5395668502897, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.5395668502897, "probs": [0.8387192579214904, 0.15415437985672864, 0.007126362221781067], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.06602098606527, "probs": [0.10592235395880453, 0.8940776460411956], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.06602098606527, "probs": [0.928892622801621, 0.06661995929549502, 0.004487417902884053], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.06602098606527, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.06602098606527, "probs": [0.9588514552888884, 0.039099387288032916, 0.002049157423078645], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.978471007198095, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.978471007198095, "probs": [0.14648079183179633, 0.8485708719675852, 0.004948336200618522], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.978471007198095, "probs": [0.9248060506082927, 0.07519394939170732], "top": "same"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.978471007198095, "probs": [0.24334788832895243, 0.7527800186067107, 0.0038720930643368974], "top": "meet_wren"}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.27355607226491, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.27355607226491, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.27355607226491, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.27355607226491, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.07200615666807, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.07200615666807, "probs": [0.8089141677995819, 0.1910858322004182], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.07200615666807, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.07200615666807, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.09394508972764, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.09394508972764, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.09394508972764, "probs": [0.9439168356016766, 0.0560831643983235], "top": "same"}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.09394508972764, "probs": [0.9156019658234928, 0.08439803417650704], "top": "stay"}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.04839503392577, "probs": [0.3201997500236374, 0.6798002499763626], "top": "holds"}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.04839503392577, "probs": [0.12511350509772579, 0.8748864949022742], "top": "departure"}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.04839503392577, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.04839503392577, "probs": [0.13214384255458153, 0.8678561574454184], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.38276801072061, "probs": [0.25606104458540857, 0.7439389554145914], "top": "holds"}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.38276801072061, "probs": [0.08439803417650704, 0.9156019658234928], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.38276801072061, "probs": [0.11840617139815589, 0.8815938286018441], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.38276801072061, "probs": [0.10012611628383591, 0.8998738837161642], "top": "departure"}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.720402013510466, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.720402013510466, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.720402013510466, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.720402013510466, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.612150898203254, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.612150898203254, "probs": [0.8184242551251981, 0.18157574487480185], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.612150898203254, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.612150898203254, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.122206004336476, "probs": [0.12511350509772579, 0.8748864949022742], "top": "holds"}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.122206004336476, "probs": [0.9248060506082927, 0.07519394939170732], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.122206004336476, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.122206004336476, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.718432964757085, "probs": [0.7193152314525416, 0.2806847685474584], "top": "moved"}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.718432964757085, "probs": [0.08937422170979005, 0.91062577829021], "top": "outside"}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.718432964757085, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.718432964757085, "probs": [0.07519394939170732, 0.9248060506082927], "top": "outside"}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.87283414043486, "probs": [0.15526082656247686, 0.844739173437523], "top": "holds"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.87283414043486, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.87283414043486, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.87283414043486, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.91530407965183, "probs": [0.45308504361568214, 0.5469149563843178], "top": "holds"}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.91530407965183, "probs": [0.25606104458540857, 0.7439389554145914], "top": "outside"}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.91530407965183, "probs": [0.15526082656247686, 0.844739173437523], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.91530407965183, "probs": [0.43758935794935333, 0.5624106420506467], "top": "outside"}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.27890917658806, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.27890917658806, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.27890917658806, "probs": [0.9203253283037116, 0.07967467169628852], "top": "same"}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.27890917658806, "probs": [0.936892368084393, 0.06310763191560703], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.50409093685448, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.50409093685448, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.50409093685448, "probs": [0.9156019658234928, 0.08439803417650704], "top": "same"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.50409093685448, "probs": [0.91062577829021, 0.08937422170979005], "top": "stay"}], "seconds": 1.0, "failures": 0}}, "finished_utc": "2026-09-30T16:14:02Z"} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/shape.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/shape.json new file mode 100644 index 0000000..57ba62f --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/shape.json @@ -0,0 +1 @@ +{"label": "r1", "backend": "semif", "runs": 3, "per": 20, "max_questions": 10000, "events": [["start", 1790784842.3553743], ["short1:measure", 1790784842.438584], ["crit21:measure", 1790784844.8261297], ["long1:measure", 1790784856.2801244], ["long16:measure", 1790784874.2582178], ["shapes:done", 1790784887.4273212], ["end", 1790784887.4273224]], "shapes": {"short1": {"rows": 1, "runs": [{"e2e_ms": [36.69911785982549, 36.558506079018116, 36.68514685705304, 36.60177602432668, 37.10522083565593, 36.61828604526818, 36.807028111070395, 36.7027570027858, 37.213773000985384, 36.48746502585709, 37.263472797349095, 36.589056020602584, 37.13556192815304, 36.56049491837621, 37.014881148934364, 36.59066581167281, 36.816218169406056, 36.78153711371124, 37.061721086502075, 37.080460926517844], "server_ms": [34.09491712227464, 33.83906395174563, 33.91585499048233, 33.65612286143005, 34.15901795960963, 33.79982407204807, 33.974275924265385, 33.810453955084085, 34.252509009093046, 33.74756406992674, 34.39380996860564, 33.56509190052748, 34.16119911707938, 33.70963199995458, 34.14635802619159, 33.63592317327857, 33.85669505223632, 33.80254399962723, 34.204788971692324, 33.83347485214472], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.74214705824852, "median_server": 33.847879501990974}, {"e2e_ms": [37.4287951271981, 37.00495092198253, 36.71695711091161, 36.82445897720754, 37.01268997974694, 36.44735412672162, 36.861378932371736, 36.85749904252589, 37.103740964084864, 36.751647014170885, 37.33838489279151, 36.607205867767334, 37.10389090701938, 36.90578998066485, 36.84925893321633, 36.812979029491544, 37.68475912511349, 36.73602710478008, 36.9455199688673, 37.098152097314596], "server_ms": [34.41481105983257, 34.19002890586853, 33.82356511428952, 33.8684250600636, 34.06613692641258, 33.645072020590305, 34.0128259267658, 34.01341591961682, 34.15512782521546, 33.868304919451475, 34.48495198972523, 33.67848298512399, 34.13621801882982, 34.107106970623136, 33.94054505042732, 33.85966503992677, 34.71853490918875, 33.85545499622822, 33.911615842953324, 34.154928056523204], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.88358445651829, "median_server": 34.01312092319131}, {"e2e_ms": [37.05438086763024, 36.54392599128187, 36.95276007056236, 36.49639501236379, 36.95848002098501, 36.57183609902859, 36.89197893254459, 36.530365934595466, 36.81097808293998, 36.408772924914956, 36.98658989742398, 36.441945005208254, 37.17020200565457, 36.9258988648653, 37.25830418989062, 36.90873901359737, 36.020959028974175, 36.33384290151298, 36.951069021597505, 36.4018629770726], "server_ms": [34.24910898320377, 33.583952113986015, 34.08557712100446, 33.69081299751997, 34.09324702806771, 33.855963964015245, 34.15172686800361, 33.7966640945524, 34.00379605591297, 33.750883070752025, 34.242888912558556, 33.63740211352706, 34.360299818217754, 34.15268799290061, 34.51938205398619, 34.59713305346668, 34.39065092243254, 33.55581103824079, 34.23589910380542, 33.71386299841106], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.851478507742286, "median_server": 34.089412074536085}], "tokens": 181, "median_e2e_ms": 36.8, "run_medians_e2e_ms": [36.7, 36.9, 36.9], "median_server_ms": 34.0, "non_200": 0}, "crit21": {"rows": 21, "runs": [{"e2e_ms": [84.12860613316298, 84.13009601645172, 84.33389803394675, 84.07884510233998, 84.42786009982228, 84.55148013308644, 83.99306400679052, 84.03353509493172, 84.33574787341058, 84.28464806638658, 84.52691091224551, 84.34583898633718, 84.02313501574099, 83.83840206079185, 84.24808783456683, 84.68699199147522, 84.45204887539148, 84.27222794853151, 84.11552617326379, 84.29796784184873], "server_ms": [77.65320292674005, 77.55638100206852, 77.8895749244839, 77.68146204762161, 77.91584497317672, 78.11973802745342, 77.52831210382283, 77.69928290508687, 77.92375492863357, 77.66886311583221, 77.96964701265097, 77.90518598631024, 77.5946518406272, 77.53113098442554, 77.9427660163492, 78.07196700014174, 77.86578498780727, 77.84204510971904, 77.8536950238049, 77.70606293343008], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 84.27843800745904, "median_server": 77.84787006676197}, {"e2e_ms": [83.93192407675087, 84.10659502260387, 83.95667397417128, 84.28796799853444, 84.17387586086988, 84.08814598806202, 84.3065578956157, 84.13612586446106, 84.19830701313913, 84.0947360266, 84.38221900723875, 84.07084620557725, 84.39509896561503, 84.58957192488015, 84.11964611150324, 84.30208708159626, 84.23383696936071, 84.5188400708139, 84.72307305783033, 84.1533369384706], "server_ms": [77.59782182984054, 77.62267300859094, 77.5662511587143, 77.88307499140501, 77.8628250118345, 77.805255074054, 77.83626392483711, 77.81131379306316, 77.78876391239464, 77.72011309862137, 78.02458596415818, 77.68469303846359, 77.77347508817911, 78.04910698905587, 77.64689181931317, 77.85133505240083, 77.76823407039046, 77.94056599959731, 78.20458896458149, 77.6689329650253], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 84.1860914370045, "median_server": 77.79700949322432}, {"e2e_ms": [84.12588597275317, 84.18751694262028, 84.46849999018013, 87.6883058808744, 85.0524757988751, 84.16745695285499, 84.16961715556681, 84.36311804689467, 84.38775991089642, 84.43822991102934, 84.46727995760739, 84.47708003222942, 84.36996815726161, 84.20524699613452, 84.33433901518583, 84.2976679559797, 84.46912094950676, 84.36188916675746, 84.35845887288451, 84.02477484196424], "server_ms": [77.65507185831666, 77.80040521174669, 77.9692460782826, 80.420833081007, 77.97554600983858, 77.69453292712569, 77.66464189626276, 77.83585507422686, 77.8651749715209, 77.8538549784571, 77.98470696434379, 78.06473691016436, 77.86531490273774, 77.63302209787071, 77.83613493666053, 77.91283493861556, 77.70676305517554, 77.77948398143053, 77.82317418605089, 77.60565215721726], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 84.36250360682607, "median_server": 77.83599500544369}], "tokens": null, "median_e2e_ms": 84.3, "run_medians_e2e_ms": [84.3, 84.2, 84.4], "median_server_ms": 77.8, "non_200": 0}, "long1": {"rows": 1, "runs": [{"e2e_ms": [187.62216181494296, 194.02699498459697, 188.42861102893949, 196.3318211492151, 185.8468430582434, 196.63205416873097, 190.01547899097204, 189.91431687027216, 188.600952969864, 195.34617010504007, 190.75165805406868, 189.11608890630305, 188.46752191893756, 191.9465009123087, 189.64864499866962, 187.25039716809988, 189.85962797887623, 190.86394901387393, 189.54728497192264, 188.9109369367361], "server_ms": [184.49826701544225, 191.2344431038946, 184.8908809479326, 192.85307102836668, 182.25402222014964, 193.08537407778203, 186.51798903010786, 186.32086785510182, 185.1254040375352, 191.74851919524372, 187.17364710755646, 185.66294107586145, 184.88497100770473, 188.42972093261778, 186.10421498306096, 183.73865797184408, 185.94143306836486, 186.94784492254257, 186.1127249430865, 185.5921889655292], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 189.75413648877293, "median_server": 186.10846996307373}, {"e2e_ms": [189.00979799218476, 197.6554160937667, 188.9446871355176, 187.25636787712574, 190.3936630114913, 192.84410099498928, 187.79134401120245, 189.22081007622182, 191.42936496064067, 196.51344302110374, 190.47133391723037, 191.45161611959338, 188.01991711370647, 195.62305393628776, 189.14339900948107, 189.13726904429495, 188.8488559052348, 193.89271410182118, 189.36585215851665, 187.63496284373105], "server_ms": [185.70771999657154, 194.34122811071575, 185.60990900732577, 183.9755210094154, 187.0533360633999, 189.54121391288936, 184.5110470894724, 185.8927619177848, 188.04113706573844, 193.20073607377708, 187.1612968388945, 188.16869799047709, 184.70818898640573, 192.32217618264258, 185.81209215335548, 185.8414621092379, 185.59491890482605, 190.55936485528946, 186.03209406137466, 184.34644490480423], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 189.29333111736923, "median_server": 185.96242798957974}, {"e2e_ms": [190.24365209043026, 195.45638095587492, 189.86437702551484, 187.75844294577837, 188.96789802238345, 196.64431479759514, 189.52532391995192, 188.17693809978664, 188.7092450633645, 192.53426790237427, 194.15852706879377, 189.0421980060637, 194.89452498964965, 189.52277419157326, 197.3768121097237, 189.8432970046997, 189.98231901787221, 190.61216595582664, 193.59390996396542, 187.98247608356178], "server_ms": [186.93562387488782, 192.1485939528793, 186.5633598063141, 184.4366060104221, 185.6803591363132, 193.31106613390148, 185.93800417147577, 184.87204099074006, 185.357796959579, 189.24867105670273, 190.84873888641596, 185.7321399729699, 191.5683269035071, 186.21404585428536, 194.02564503252506, 186.58776092343032, 186.666440917179, 187.38676887005568, 190.2653118595481, 184.62813808582723], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 189.92334802169353, "median_server": 186.62710092030466}], "tokens": 4019, "median_e2e_ms": 189.9, "run_medians_e2e_ms": [189.8, 189.3, 189.9], "median_server_ms": 186.3, "non_200": 0}, "long16": {"rows": 16, "runs": [{"e2e_ms": [219.12985993549228, 215.7682019751519, 218.0578380357474, 220.86331993341446, 222.5807891227305, 213.97431194782257, 219.39542284235358, 219.20029097236693, 217.25444914773107, 217.22338791005313, 217.3743809107691, 225.1270581036806, 221.07367194257677, 217.57508185692132, 218.03153795190156, 224.17462710291147, 220.82681907340884, 219.57236505113542, 219.14622001349926, 216.65072184987366], "server_ms": [213.3143439423293, 209.96249513700604, 212.2595210094005, 215.04524303600192, 216.55616094358265, 208.08226522058249, 213.5846468154341, 213.39214500039816, 211.4139029290527, 211.45591302774847, 211.59638394601643, 219.28260196000338, 215.36155813373625, 211.70442597940564, 212.24042191170156, 218.33051112480462, 215.03918315283954, 213.81478011608124, 212.78379810974002, 210.3766999207437], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 219.13803997449577, "median_server": 213.04907102603465}, {"e2e_ms": [214.5290281623602, 217.64561301097274, 218.5149029828608, 216.2398281507194, 220.29660316184163, 218.52417290210724, 217.93758706189692, 217.45889214798808, 214.55801813863218, 215.42864898219705, 215.38862702436745, 219.2836320027709, 219.43697286769748, 221.16683307103813, 220.35969397984445, 213.16217305138707, 219.85806780867279, 220.80282890237868, 219.5669850334525, 217.76637504808605], "server_ms": [208.75647314824164, 211.87490806914866, 212.62732590548694, 210.47035208903253, 214.52508796937764, 212.78268797323108, 212.16092095710337, 211.7764470167458, 208.87030381709337, 209.63216200470924, 209.74833401851356, 213.64131709560752, 213.72481808066368, 215.54838889278471, 214.69976007938385, 207.41148688830435, 214.07895302399993, 214.93555302731693, 213.91782094724476, 212.06530020572245], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 218.22624502237886, "median_server": 212.39412343129516}, {"e2e_ms": [220.08436103351414, 221.8698209617287, 224.16663798503578, 224.12922699004412, 221.75859985873103, 222.11640398018062, 223.7041520420462, 225.31324019655585, 218.33116095513105, 213.71859894134104, 219.10095890052617, 218.08090806007385, 214.32480588555336, 217.5051518715918, 226.17439995519817, 217.44852093979716, 224.4490310549736, 216.72601206228137, 224.8746759723872, 218.73442502692342], "server_ms": [214.2970240674913, 216.13611583597958, 218.49283296614885, 218.48959289491177, 216.02780488319695, 216.46389085799456, 217.88247604854405, 219.52079492621124, 212.56327512674034, 208.02635396830738, 213.46307499334216, 212.38096384331584, 208.6048610508442, 211.87746687792242, 220.4376459121704, 211.8128479924053, 218.69937586598098, 210.98208706825972, 219.15082004852593, 212.92339893989265], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 220.9214804461226, "median_server": 215.16241447534412}], "tokens": null, "median_e2e_ms": 219.1, "run_medians_e2e_ms": [219.1, 218.2, 220.9], "median_server_ms": 213.1, "non_200": 0}}} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/vram.csv new file mode 100644 index 0000000..c17985b --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1-pre-thread-fix/vram.csv @@ -0,0 +1,1490 @@ +# pid=1826826 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790784746.449897071,8812,8822 +1790784746.549960488,8812,8822 +1790784746.654036051,8812,8822 +1790784746.765060782,8812,8822 +1790784746.870703453,8812,8822 +1790784746.976156641,8812,8822 +1790784747.085211900,8812,8822 +1790784747.190503597,8812,8822 +1790784747.297901587,8812,8822 +1790784747.403370506,8812,8822 +1790784747.508171177,8812,8822 +1790784747.617687842,8812,8822 +1790784747.720883984,8812,8822 +1790784747.828899741,8812,8822 +1790784747.935993409,8812,8822 +1790784748.041453017,8812,8822 +1790784748.146325769,8812,8822 +1790784748.253917851,8812,8822 +1790784748.360536954,8812,8822 +1790784748.468406839,8812,8822 +1790784748.576368816,8812,8822 +1790784748.680920444,8812,8822 +1790784748.788133702,8812,8822 +1790784748.896544464,8812,8822 +1790784748.998769996,8812,8822 +1790784749.098133585,8812,8822 +1790784749.204781718,8812,8822 +1790784749.311405619,8812,8822 +1790784749.414877675,8846,8856 +1790784749.519678285,8846,8856 +1790784749.624185073,8846,8856 +1790784749.731973168,8846,8856 +1790784749.840366990,8846,8856 +1790784749.948129715,8846,8856 +1790784750.057164954,8846,8856 +1790784750.167562558,8846,8856 +1790784750.272249447,8846,8856 +1790784750.382184407,8846,8856 +1790784750.492403990,8846,8856 +1790784750.599832510,8846,8856 +1790784750.706090358,8846,8856 +1790784750.815700693,8846,8856 +1790784750.925071896,8846,8856 +1790784751.032535197,8846,8856 +1790784751.140362213,8846,8856 +1790784751.247079995,8846,8856 +1790784751.351109457,8846,8856 +1790784751.456222512,8846,8856 +1790784751.564762036,8846,8856 +1790784751.671706981,8846,8856 +1790784751.775748003,8846,8856 +1790784751.882926461,8846,8856 +1790784751.987967655,8846,8856 +1790784752.097048434,8846,8856 +1790784752.203329752,8846,8856 +1790784752.308598758,8846,8856 +1790784752.416330912,8846,8856 +1790784752.524246539,8846,8856 +1790784752.632051024,8846,8856 +1790784752.741582829,8846,8856 +1790784752.848335092,8846,8856 +1790784752.955580241,8846,8856 +1790784753.060878427,8846,8856 +1790784753.169161698,8846,8856 +1790784753.275862180,8846,8856 +1790784753.384048169,8846,8856 +1790784753.488515157,8846,8856 +1790784753.597888490,8846,8856 +1790784753.704817445,8846,8856 +1790784753.814077946,8846,8856 +1790784753.919830148,8846,8856 +1790784754.028076118,8846,8856 +1790784754.135508249,8846,8856 +1790784754.244294165,8846,8856 +1790784754.352395174,8846,8856 +1790784754.458216736,8846,8856 +1790784754.566020671,8846,8856 +1790784754.675666757,8846,8856 +1790784754.783021017,8846,8856 +1790784754.891878084,8846,8856 +1790784755.001208656,8846,8856 +1790784755.112554042,8846,8856 +1790784755.221409479,8846,8856 +1790784755.329604359,8846,8856 +1790784755.437448014,8846,8856 +1790784755.544282248,8846,8856 +1790784755.655775915,8846,8856 +1790784755.768435135,8846,8856 +1790784755.876655855,8846,8856 +1790784755.985329410,8846,8856 +1790784756.094184327,8846,8856 +1790784756.201873631,8846,8856 +1790784756.307109157,8846,8856 +1790784756.416716352,8846,8856 +1790784756.523952951,8846,8856 +1790784756.628535009,8846,8856 +1790784756.733928407,8846,8856 +1790784756.835741054,8846,8856 +1790784756.949458786,8846,8856 +1790784757.057274871,8846,8856 +1790784757.168615616,8846,8856 +1790784757.277879158,8846,8856 +1790784757.381632337,8846,8856 +1790784757.492967012,8846,8856 +1790784757.601213532,8846,8856 +1790784757.709150949,8846,8856 +1790784757.817524951,8846,8856 +1790784757.925359626,8846,8856 +1790784758.033188261,8846,8856 +1790784758.141675764,8846,8856 +1790784758.252716156,8846,8856 +1790784758.362101629,8846,8856 +1790784758.466046560,8846,8856 +1790784758.571096174,8846,8856 +1790784758.679032551,8846,8856 +1790784758.785755734,8846,8856 +1790784758.892039751,8846,8856 +1790784758.999943057,8846,8856 +1790784759.109677824,8846,8856 +1790784759.219975028,8846,8856 +1790784759.329380241,8846,8856 +1790784759.436760531,8846,8856 +1790784759.544576216,8846,8856 +1790784759.650966695,8846,8856 +1790784759.759238746,8846,8856 +1790784759.868322076,8846,8856 +1790784759.977515756,8846,8856 +1790784760.087674308,8846,8856 +1790784760.198167734,8846,8856 +1790784760.308098143,8846,8856 +1790784760.417765129,8846,8856 +1790784760.526988081,8846,8856 +1790784760.636310663,8846,8856 +1790784760.745394132,8846,8856 +1790784760.852844293,8846,8856 +1790784760.959508966,8846,8856 +1790784761.067302201,8846,8856 +1790784761.172398765,8846,8856 +1790784761.279160188,8846,8856 +1790784761.385427435,8846,8856 +1790784761.492983698,8846,8856 +1790784761.599690760,8846,8856 +1790784761.708361675,8846,8856 +1790784761.816640526,8846,8856 +1790784761.924286469,8846,8856 +1790784762.034006526,8846,8856 +1790784762.144201338,8846,8856 +1790784762.253556331,8846,8856 +1790784762.359405114,8846,8856 +1790784762.465821153,8846,8856 +1790784762.571264381,8846,8856 +1790784762.679938466,8846,8856 +1790784762.785080471,8846,8856 +1790784762.892975887,8846,8856 +1790784763.000187346,8846,8856 +1790784763.108270974,8846,8856 +1790784763.219404026,8846,8856 +1790784763.325309530,8846,8856 +1790784763.431952262,8846,8856 +1790784763.539118920,8846,8856 +1790784763.647038146,8846,8856 +1790784763.753065711,8846,8856 +1790784763.861724686,8846,8856 +1790784763.967767091,8866,8876 +1790784764.069702109,8866,8876 +1790784764.179646519,8866,8876 +1790784764.287346072,8866,8876 +1790784764.389068939,8866,8876 +1790784764.493737038,8866,8876 +1790784764.602004448,8866,8876 +1790784764.711513833,8866,8876 +1790784764.816171192,8866,8876 +1790784764.922960956,8866,8876 +1790784765.029566637,8866,8876 +1790784765.136575113,8866,8876 +1790784765.243870162,8866,8876 +1790784765.351354274,8866,8876 +1790784765.455178944,8866,8876 +1790784765.563287022,8866,8876 +1790784765.667877891,8866,8876 +1790784765.776787268,8866,8876 +1790784765.884104998,8866,8876 +1790784765.989853339,8866,8876 +1790784766.095560741,8866,8876 +1790784766.199235609,8866,8876 +1790784766.309772735,8866,8876 +1790784766.419394891,8866,8876 +1790784766.529169948,8866,8876 +1790784766.638432700,8866,8876 +1790784766.750069488,8866,8876 +1790784766.859786755,8866,8876 +1790784766.967800553,8866,8876 +1790784767.073647455,8866,8876 +1790784767.177504476,8866,8876 +1790784767.280412265,8866,8876 +1790784767.387778545,8866,8876 +1790784767.493850540,8866,8876 +1790784767.601258201,8866,8876 +1790784767.713073002,8866,8876 +1790784767.823319864,8866,8876 +1790784767.931331992,8866,8876 +1790784768.041453433,8866,8876 +1790784768.150390671,8866,8876 +1790784768.260107428,8866,8876 +1790784768.368298117,8866,8876 +1790784768.476919992,8866,8876 +1790784768.583875527,8866,8876 +1790784768.689545278,8866,8876 +1790784768.796339522,8866,8876 +1790784768.901601668,8866,8876 +1790784769.009063369,8866,8876 +1790784769.116306688,8866,8876 +1790784769.225195525,8866,8876 +1790784769.335383497,8866,8876 +1790784769.441694456,8866,8876 +1790784769.552609946,8866,8876 +1790784769.664675610,8866,8876 +1790784769.776681562,8866,8876 +1790784769.889092550,8866,8876 +1790784769.999641656,8866,8876 +1790784770.104134403,8866,8876 +1790784770.212343413,8866,8876 +1790784770.320215119,8866,8876 +1790784770.429218108,8866,8876 +1790784770.536132722,8866,8876 +1790784770.645162171,8866,8876 +1790784770.753146839,8866,8876 +1790784770.856408362,8866,8876 +1790784770.962949113,8866,8876 +1790784771.073419768,8866,8876 +1790784771.180309123,8866,8876 +1790784771.291269714,8866,8876 +1790784771.397615072,8866,8876 +1790784771.503340294,8866,8876 +1790784771.605222101,8866,8876 +1790784771.713167798,8866,8876 +1790784771.819725959,8866,8876 +1790784771.929306054,8866,8876 +1790784772.037464913,8866,8876 +1790784772.140689467,8866,8876 +1790784772.247015485,8866,8876 +1790784772.348497268,8866,8876 +1790784772.450973503,8866,8876 +1790784772.553427497,8866,8876 +1790784772.659998838,8866,8876 +1790784772.770026848,8866,8876 +1790784772.879175338,8866,8876 +1790784772.984173282,8866,8876 +1790784773.088769620,8866,8876 +1790784773.191286695,8866,8876 +1790784773.295216746,8866,8876 +1790784773.399999297,8866,8876 +1790784773.504053639,8866,8876 +1790784773.612419731,8866,8876 +1790784773.719116183,8866,8876 +1790784773.825294650,8866,8876 +1790784773.933525960,8866,8876 +1790784774.047097930,8866,8876 +1790784774.149121020,8866,8876 +1790784774.257783225,8866,8876 +1790784774.364343035,8866,8876 +1790784774.469781214,8866,8876 +1790784774.577445607,8866,8876 +1790784774.684565695,8866,8876 +1790784774.792522331,8866,8876 +1790784774.901091305,8866,8876 +1790784775.004147576,8866,8876 +1790784775.109872497,8866,8876 +1790784775.216700672,8866,8876 +1790784775.321736515,8866,8876 +1790784775.423758335,8866,8876 +1790784775.532207927,8866,8876 +1790784775.639917731,8866,8876 +1790784775.747957129,8866,8876 +1790784775.855776364,8866,8876 +1790784775.965335829,8866,8876 +1790784776.071301233,8866,8876 +1790784776.172825737,8866,8876 +1790784776.277684068,8866,8876 +1790784776.384498502,8866,8876 +1790784776.496481395,8866,8876 +1790784776.608131853,8866,8876 +1790784776.717537127,8866,8876 +1790784776.826142181,8866,8876 +1790784776.933679893,8866,8876 +1790784777.037767446,8866,8876 +1790784777.143140583,8866,8876 +1790784777.251362513,8866,8876 +1790784777.357063714,8866,8876 +1790784777.465270264,8866,8876 +1790784777.575517757,8866,8876 +1790784777.685116802,8866,8876 +1790784777.792932988,8866,8876 +1790784777.897718398,8866,8876 +1790784778.004316150,8866,8876 +1790784778.113034135,8866,8876 +1790784778.220110962,8866,8876 +1790784778.332649811,8866,8876 +1790784778.436975186,8866,8876 +1790784778.545283247,8866,8876 +1790784778.649328490,8866,8876 +1790784778.758020224,8866,8876 +1790784778.863234740,8866,8876 +1790784778.964116336,8866,8876 +1790784779.071428226,8866,8876 +1790784779.175450648,8866,8876 +1790784779.279921515,8866,8876 +1790784779.383796696,8866,8876 +1790784779.487503214,8866,8876 +1790784779.595680803,8866,8876 +1790784779.701960011,8866,8876 +1790784779.813634540,8866,8876 +1790784779.924366149,8866,8876 +1790784780.032025662,8866,8876 +1790784780.139479233,8866,8876 +1790784780.248580283,8866,8876 +1790784780.356696482,8866,8876 +1790784780.464178863,8866,8876 +1790784780.572694036,8866,8876 +1790784780.680255079,8866,8876 +1790784780.788878433,8866,8876 +1790784780.898035953,8866,8876 +1790784781.008423528,8866,8876 +1790784781.112846225,8866,8876 +1790784781.223375381,8866,8876 +1790784781.334085598,8866,8876 +1790784781.443325120,8866,8876 +1790784781.553082827,8866,8876 +1790784781.661814193,8866,8876 +1790784781.773194778,8866,8876 +1790784781.881887613,8866,8876 +1790784781.991080434,8866,8876 +1790784782.099881211,8866,8876 +1790784782.209289744,8866,8876 +1790784782.317962699,8866,8876 +1790784782.426697344,8866,8876 +1790784782.536066947,8866,8876 +1790784782.647022878,8866,8876 +1790784782.757587755,8866,8876 +1790784782.864134686,8866,8876 +1790784782.977213891,8866,8876 +1790784783.084677912,8866,8876 +1790784783.194726482,8866,8876 +1790784783.301588037,8866,8876 +1790784783.409457143,8866,8876 +1790784783.516913094,8866,8876 +1790784783.621779235,8866,8876 +1790784783.727272194,8866,8876 +1790784783.833771274,8866,8876 +1790784783.942077465,8866,8876 +1790784784.051978484,8866,8876 +1790784784.158855388,8866,8876 +1790784784.265260878,8866,8876 +1790784784.372972682,8866,8876 +1790784784.479616234,8866,8876 +1790784784.588849935,8866,8876 +1790784784.698785434,8866,8876 +1790784784.809413981,8866,8876 +1790784784.917919374,8866,8876 +1790784785.028513871,8866,8876 +1790784785.139008607,8866,8876 +1790784785.249722305,8866,8876 +1790784785.361615947,8866,8876 +1790784785.471976661,8866,8876 +1790784785.579923447,8866,8876 +1790784785.688749184,8866,8876 +1790784785.797915875,8866,8876 +1790784785.904169162,8866,8876 +1790784786.012423432,8866,8876 +1790784786.120508791,8866,8876 +1790784786.229239696,8866,8876 +1790784786.336484585,8866,8876 +1790784786.447701158,8866,8876 +1790784786.556425874,8866,8876 +1790784786.662987425,8866,8876 +1790784786.767805366,8866,8876 +1790784786.872968871,8866,8876 +1790784786.981498024,8866,8876 +1790784787.089243999,8866,8876 +1790784787.199104827,8866,8876 +1790784787.309987688,8866,8876 +1790784787.421967940,8866,8876 +1790784787.530077088,8866,8876 +1790784787.637286187,8866,8876 +1790784787.745330744,8866,8876 +1790784787.856155994,8866,8876 +1790784787.968595962,8866,8876 +1790784788.078876995,8866,8876 +1790784788.188461370,8866,8876 +1790784788.298438060,8866,8876 +1790784788.404060470,8866,8876 +1790784788.507979361,8866,8876 +1790784788.613511810,8866,8876 +1790784788.725529863,8866,8876 +1790784788.835910668,8866,8876 +1790784788.942223276,8866,8876 +1790784789.053151236,8866,8876 +1790784789.160552657,8866,8876 +1790784789.267730415,8866,8876 +1790784789.375335328,8866,8876 +1790784789.484280986,8866,8876 +1790784789.593088242,8866,8876 +1790784789.701525664,8866,8876 +1790784789.808715172,8866,8876 +1790784789.915439105,8866,8876 +1790784790.022062907,8866,8876 +1790784790.129738331,8866,8876 +1790784790.240129035,8866,8876 +1790784790.347235962,8866,8876 +1790784790.455460372,8866,8876 +1790784790.563375998,8866,8876 +1790784790.671319405,8866,8876 +1790784790.779449354,8866,8876 +1790784790.889178981,8866,8876 +1790784790.999193991,8866,8876 +1790784791.107370400,8866,8876 +1790784791.214052513,8866,8876 +1790784791.322203171,8866,8876 +1790784791.428904644,8866,8876 +1790784791.536995442,8866,8876 +1790784791.644982909,8866,8876 +1790784791.753069978,8866,8876 +1790784791.860645280,8866,8876 +1790784791.967355003,8866,8876 +1790784792.072807301,8866,8876 +1790784792.182969913,8866,8876 +1790784792.289155240,8866,8876 +1790784792.396042494,8866,8876 +1790784792.499918965,8866,8876 +1790784792.608253356,8866,8876 +1790784792.716790119,8866,8876 +1790784792.823690884,8866,8876 +1790784792.933231439,8866,8876 +1790784793.039220533,8866,8876 +1790784793.147076379,8866,8876 +1790784793.256887807,8866,8876 +1790784793.364929555,8866,8876 +1790784793.471545066,8866,8876 +1790784793.579950898,8866,8876 +1790784793.688044946,8866,8876 +1790784793.794604747,8866,8876 +1790784793.901736005,8866,8876 +1790784794.009137605,8866,8876 +1790784794.116385444,8866,8876 +1790784794.222473910,8866,8876 +1790784794.328465454,8866,8876 +1790784794.435642752,8866,8876 +1790784794.540031418,8866,8876 +1790784794.643860868,8866,8876 +1790784794.750736862,8866,8876 +1790784794.854039137,8866,8876 +1790784794.957560423,8866,8876 +1790784795.061854368,8866,8876 +1790784795.165058661,8866,8876 +1790784795.269587219,8866,8876 +1790784795.374117527,8866,8876 +1790784795.479624265,8866,8876 +1790784795.583167772,8866,8876 +1790784795.691481323,8866,8876 +1790784795.799490471,8866,8876 +1790784795.906334144,8866,8876 +1790784796.016477806,8866,8876 +1790784796.124474593,8866,8876 +1790784796.232375140,8866,8876 +1790784796.336327381,8866,8876 +1790784796.443285847,8866,8876 +1790784796.548456611,8866,8876 +1790784796.655627410,8866,8876 +1790784796.761746425,8866,8876 +1790784796.866206992,8866,8876 +1790784796.969436375,8866,8876 +1790784797.080222494,8866,8876 +1790784797.192109326,8866,8876 +1790784797.300893072,8866,8876 +1790784797.406715615,8866,8876 +1790784797.515239258,8866,8876 +1790784797.620531385,8866,8876 +1790784797.727937415,8866,8876 +1790784797.835279345,8866,8876 +1790784797.945856452,8866,8876 +1790784798.058096317,8866,8876 +1790784798.164452016,8866,8876 +1790784798.271231569,8866,8876 +1790784798.378692240,8866,8876 +1790784798.485808837,8866,8876 +1790784798.593106086,8866,8876 +1790784798.703020805,8866,8876 +1790784798.807462023,8866,8876 +1790784798.912426585,8866,8876 +1790784799.020836567,8866,8876 +1790784799.128328469,8866,8876 +1790784799.237387198,8866,8876 +1790784799.345146352,8866,8876 +1790784799.452886917,8866,8876 +1790784799.555922798,8866,8876 +1790784799.661449337,8866,8876 +1790784799.769858459,8866,8876 +1790784799.878706346,8866,8876 +1790784799.987743335,8866,8876 +1790784800.095400628,8866,8876 +1790784800.200936597,8866,8876 +1790784800.308185947,8866,8876 +1790784800.417552499,8866,8876 +1790784800.524640756,8866,8876 +1790784800.636688499,8866,8876 +1790784800.745643777,8866,8876 +1790784800.856230574,8866,8876 +1790784800.960403428,8866,8876 +1790784801.065961268,8866,8876 +1790784801.173142506,8866,8876 +1790784801.281705519,8866,8876 +1790784801.391068272,8866,8876 +1790784801.500555696,8866,8876 +1790784801.611186303,8866,8876 +1790784801.720089211,8866,8876 +1790784801.828355861,8866,8876 +1790784801.937058377,8866,8876 +1790784802.043642148,8866,8876 +1790784802.152800159,8866,8876 +1790784802.261777977,8866,8876 +1790784802.370929117,8866,8876 +1790784802.473848847,8866,8876 +1790784802.579885191,8866,8876 +1790784802.687935109,8866,8876 +1790784802.793639391,8866,8876 +1790784802.901093282,8866,8876 +1790784803.008884046,8866,8876 +1790784803.116623641,8866,8876 +1790784803.226581601,8866,8876 +1790784803.336882934,8866,8876 +1790784803.440149978,8866,8876 +1790784803.547534318,8866,8876 +1790784803.657050892,8866,8876 +1790784803.762682333,8866,8876 +1790784803.866266230,8866,8876 +1790784803.975683743,8866,8876 +1790784804.083975824,8866,8876 +1790784804.191692658,8866,8876 +1790784804.298919086,8866,8876 +1790784804.403954310,8866,8876 +1790784804.510949206,8866,8876 +1790784804.614064038,8866,8876 +1790784804.720873391,8866,8876 +1790784804.829806539,8866,8876 +1790784804.936510532,8866,8876 +1790784805.043823351,8866,8876 +1790784805.155005235,8866,8876 +1790784805.264971094,8866,8876 +1790784805.373533238,8866,8876 +1790784805.480367412,8866,8876 +1790784805.588636952,8866,8876 +1790784805.700906648,8866,8876 +1790784805.809882317,8866,8876 +1790784805.920708846,8866,8876 +1790784806.030535994,8866,8876 +1790784806.140032128,8866,8876 +1790784806.245937972,8866,8876 +1790784806.353967929,8866,8876 +1790784806.463401303,8866,8876 +1790784806.569274916,8866,8876 +1790784806.675960688,8866,8876 +1790784806.783617462,8866,8876 +1790784806.890118832,8866,8876 +1790784806.997305460,8866,8876 +1790784807.103896172,8866,8876 +1790784807.213218314,8866,8876 +1790784807.320138969,8866,8876 +1790784807.427601810,8866,8876 +1790784807.535345755,8866,8876 +1790784807.640982075,8866,8876 +1790784807.747747168,8866,8876 +1790784807.857107621,8866,8876 +1790784807.960172352,8866,8876 +1790784808.068626865,8866,8876 +1790784808.175457899,8866,8876 +1790784808.281482804,8866,8876 +1790784808.387900783,8866,8876 +1790784808.496157763,8866,8876 +1790784808.605082471,8866,8876 +1790784808.713383482,8866,8876 +1790784808.820939924,8866,8876 +1790784808.927972240,8866,8876 +1790784809.035941127,8866,8876 +1790784809.145698784,8866,8876 +1790784809.256151000,8866,8876 +1790784809.359513924,8866,8876 +1790784809.471477296,8866,8876 +1790784809.573806279,8866,8876 +1790784809.680306510,8866,8876 +1790784809.788968584,8866,8876 +1790784809.896556917,8866,8876 +1790784810.005212901,8866,8876 +1790784810.110043873,8866,8876 +1790784810.212444797,8866,8876 +1790784810.319958398,8866,8876 +1790784810.422644965,8866,8876 +1790784810.526713858,8866,8876 +1790784810.631512799,8866,8876 +1790784810.737730246,8866,8876 +1790784810.842817890,8866,8876 +1790784810.948055976,8866,8876 +1790784811.055211194,8866,8876 +1790784811.159862703,8866,8876 +1790784811.266897129,8866,8876 +1790784811.373083596,8866,8876 +1790784811.477005427,8866,8876 +1790784811.587726855,8866,8876 +1790784811.693919592,8866,8876 +1790784811.801106720,8866,8876 +1790784811.907951224,8866,8876 +1790784812.011351439,8866,8876 +1790784812.119733581,8866,8876 +1790784812.229196245,8866,8876 +1790784812.339070263,8866,8876 +1790784812.448136792,8866,8876 +1790784812.555043237,8866,8876 +1790784812.661360195,8866,8876 +1790784812.768028398,8866,8876 +1790784812.874067303,8866,8876 +1790784812.981055138,8866,8876 +1790784813.088191746,8866,8876 +1790784813.196399756,8866,8876 +1790784813.305377014,8866,8876 +1790784813.410066824,8866,8876 +1790784813.515528892,8866,8876 +1790784813.622156274,8866,8876 +1790784813.728615384,8866,8876 +1790784813.836526900,8866,8876 +1790784813.944658778,8866,8876 +1790784814.053613737,8866,8876 +1790784814.157433216,8866,8876 +1790784814.265174861,8866,8876 +1790784814.372288278,8866,8876 +1790784814.477325872,8866,8876 +1790784814.588803248,8866,8876 +1790784814.699595918,8866,8876 +1790784814.809117862,8866,8876 +1790784814.916781096,8866,8876 +1790784815.023025133,8866,8876 +1790784815.133007393,8866,8876 +1790784815.241613897,8866,8876 +1790784815.346873393,8866,8876 +1790784815.452444943,8866,8876 +1790784815.556163011,8866,8876 +1790784815.663350310,8866,8876 +1790784815.771780572,8866,8876 +1790784815.879916760,8866,8876 +1790784815.988838948,8866,8876 +1790784816.095663702,8866,8876 +1790784816.200453523,8866,8876 +1790784816.307789273,8866,8876 +1790784816.415504197,8866,8876 +1790784816.524539176,8866,8876 +1790784816.629960124,8866,8876 +1790784816.737507546,8866,8876 +1790784816.845158019,8866,8876 +1790784816.953303299,8866,8876 +1790784817.061087633,8866,8876 +1790784817.166400890,8866,8876 +1790784817.271177581,8866,8876 +1790784817.377671061,8866,8876 +1790784817.484629486,8866,8876 +1790784817.589145714,8866,8876 +1790784817.693806753,8866,8876 +1790784817.800888880,8866,8876 +1790784817.908522403,8866,8876 +1790784818.015850063,8866,8876 +1790784818.122802868,8866,8876 +1790784818.229221398,8866,8876 +1790784818.336432876,8866,8876 +1790784818.441359628,8866,8876 +1790784818.548003480,8866,8876 +1790784818.653609880,8866,8876 +1790784818.761563267,8866,8876 +1790784818.871481716,8866,8876 +1790784818.977996406,8866,8876 +1790784819.088934277,8866,8876 +1790784819.200246472,8866,8876 +1790784819.309166130,8866,8876 +1790784819.417367810,8866,8876 +1790784819.528228799,8866,8876 +1790784819.636995445,8866,8876 +1790784819.749070309,8866,8876 +1790784819.861045722,8866,8876 +1790784819.970076881,8866,8876 +1790784820.080268113,8866,8876 +1790784820.188002977,8866,8876 +1790784820.294776900,8866,8876 +1790784820.399999006,8866,8876 +1790784820.508125455,8866,8876 +1790784820.614664336,8866,8876 +1790784820.721205256,8866,8876 +1790784820.832003005,8866,8876 +1790784820.940366566,8866,8876 +1790784821.051336228,8866,8876 +1790784821.159060062,8866,8876 +1790784821.269009421,8866,8876 +1790784821.380356407,8866,8876 +1790784821.488406894,8866,8876 +1790784821.597883438,8866,8876 +1790784821.705010506,8866,8876 +1790784821.813973224,8866,8876 +1790784821.923154945,8866,8876 +1790784822.032921592,8866,8876 +1790784822.144211077,8866,8876 +1790784822.253342887,8866,8876 +1790784822.363448198,8866,8876 +1790784822.471344924,8866,8876 +1790784822.580792008,8866,8876 +1790784822.689731736,8866,8876 +1790784822.797546321,8866,8876 +1790784822.902829447,8866,8876 +1790784823.012058629,8866,8876 +1790784823.121203449,8866,8876 +1790784823.228837052,8866,8876 +1790784823.337740010,8866,8876 +1790784823.445199331,8866,8876 +1790784823.551337557,8866,8876 +1790784823.654629031,8866,8876 +1790784823.760795717,8866,8876 +1790784823.867212067,8866,8876 +1790784823.973362983,8866,8876 +1790784824.077088642,8866,8876 +1790784824.185095949,8866,8876 +1790784824.296523455,8866,8876 +1790784824.403434530,8866,8876 +1790784824.508554565,8866,8876 +1790784824.616681293,8866,8876 +1790784824.729637117,8866,8876 +1790784824.838992230,8866,8876 +1790784824.946955917,8866,8876 +1790784825.052831990,8866,8876 +1790784825.158878465,8866,8876 +1790784825.268468851,8866,8876 +1790784825.377021494,8866,8876 +1790784825.485343535,8866,8876 +1790784825.590666782,8866,8876 +1790784825.696472844,8866,8876 +1790784825.809202625,8866,8876 +1790784825.916462324,8866,8876 +1790784826.025753026,8866,8876 +1790784826.130920001,8866,8876 +1790784826.237699004,8866,8876 +1790784826.342096301,8866,8876 +1790784826.445282594,8866,8876 +1790784826.551268778,8866,8876 +1790784826.655853136,8866,8876 +1790784826.760804069,8866,8876 +1790784826.866744853,8866,8876 +1790784826.971284691,8866,8876 +1790784827.075913390,8866,8876 +1790784827.181872964,8866,8876 +1790784827.286434962,8866,8876 +1790784827.391747119,8866,8876 +1790784827.499745676,8866,8876 +1790784827.608075617,8866,8876 +1790784827.721467556,8866,8876 +1790784827.832334236,8866,8876 +1790784827.944088605,8866,8876 +1790784828.055707034,8866,8876 +1790784828.165226219,8866,8876 +1790784828.274006125,8866,8876 +1790784828.380911989,8866,8876 +1790784828.491181852,8866,8876 +1790784828.601259793,8866,8876 +1790784828.714065995,8866,8876 +1790784828.821937811,8866,8876 +1790784828.929127349,8866,8876 +1790784829.050153845,8866,8876 +1790784829.155666834,8866,8876 +1790784829.266482813,8866,8876 +1790784829.371596147,8866,8876 +1790784829.481608918,8866,8876 +1790784829.590098850,8866,8876 +1790784829.698175129,8866,8876 +1790784829.809715436,8866,8876 +1790784829.918952367,8866,8876 +1790784830.027552432,8866,8876 +1790784830.135957243,8866,8876 +1790784830.245800232,8866,8876 +1790784830.357681423,8866,8876 +1790784830.469360972,8866,8876 +1790784830.579219560,8866,8876 +1790784830.688768505,8866,8876 +1790784830.796138516,8866,8876 +1790784830.903764529,8866,8876 +1790784831.014450336,8866,8876 +1790784831.117515488,8866,8876 +1790784831.227690090,8866,8876 +1790784831.336021161,8866,8876 +1790784831.445182491,8866,8876 +1790784831.551303937,8866,8876 +1790784831.661734192,8866,8876 +1790784831.770740961,8866,8876 +1790784831.882213308,8866,8876 +1790784831.991579080,8866,8876 +1790784832.101844413,8866,8876 +1790784832.209659448,8866,8876 +1790784832.317983800,8866,8876 +1790784832.421666038,8866,8876 +1790784832.528639964,8866,8876 +1790784832.640001659,8866,8876 +1790784832.748954857,8866,8876 +1790784832.853733638,8866,8876 +1790784832.961001147,8866,8876 +1790784833.068991824,8866,8876 +1790784833.176281613,8866,8876 +1790784833.285823328,8866,8876 +1790784833.389431565,8866,8876 +1790784833.496367540,8866,8876 +1790784833.604473829,8866,8876 +1790784833.713743111,8866,8876 +1790784833.815262005,8866,8876 +1790784833.923159220,8866,8876 +1790784834.027876770,8866,8876 +1790784834.137224343,8866,8876 +1790784834.245384842,8866,8876 +1790784834.351655990,8866,8876 +1790784834.458629286,8866,8876 +1790784834.563238914,8866,8876 +1790784834.669771375,8866,8876 +1790784834.778128596,8866,8876 +1790784834.891861699,8866,8876 +1790784835.004480359,8866,8876 +1790784835.114783182,8866,8876 +1790784835.226530602,8866,8876 +1790784835.336220188,8866,8876 +1790784835.447999529,8866,8876 +1790784835.564080308,8866,8876 +1790784835.673685763,8866,8876 +1790784835.785529624,8866,8876 +1790784835.895544924,8866,8876 +1790784836.004951988,8866,8876 +1790784836.112314258,8866,8876 +1790784836.221688161,8866,8876 +1790784836.331510598,8866,8876 +1790784836.437701375,8866,8876 +1790784836.548915579,8866,8876 +1790784836.658683916,8866,8876 +1790784836.767148279,8866,8876 +1790784836.875880565,8866,8876 +1790784836.985834274,8866,8876 +1790784837.092532207,8866,8876 +1790784837.200754007,8866,8876 +1790784837.311885109,8866,8876 +1790784837.415008671,8866,8876 +1790784837.521002196,8866,8876 +1790784837.629383847,8866,8876 +1790784837.739659810,8866,8876 +1790784837.847981921,8866,8876 +1790784837.956105170,8866,8876 +1790784838.064658034,8866,8876 +1790784838.168964929,8866,8876 +1790784838.275422189,8866,8876 +1790784838.376766470,8866,8876 +1790784838.484412614,8866,8876 +1790784838.590998105,8866,8876 +1790784838.697644357,8866,8876 +1790784838.803612031,8866,8876 +1790784838.908069128,8866,8876 +1790784839.016218197,8866,8876 +1790784839.125159465,8866,8876 +1790784839.235318877,8866,8876 +1790784839.345246206,8866,8876 +1790784839.452247492,8866,8876 +1790784839.560717914,8866,8876 +1790784839.669468641,8866,8876 +1790784839.776840541,8866,8876 +1790784839.883463772,8866,8876 +1790784839.990840662,8866,8876 +1790784840.096351541,8866,8876 +1790784840.204866624,8866,8876 +1790784840.314699422,8866,8876 +1790784840.420544056,8866,8876 +1790784840.528416071,8866,8876 +1790784840.636873614,8866,8876 +1790784840.747462850,8866,8876 +1790784840.856504199,8866,8876 +1790784840.965314226,8866,8876 +1790784841.072563695,8866,8876 +1790784841.180385390,8866,8876 +1790784841.283950257,8866,8876 +1790784841.392109616,8940,8950 +1790784841.501340667,8940,8950 +1790784841.612026525,8940,8950 +1790784841.720813122,8940,8950 +1790784841.829123692,8940,8950 +1790784841.936549173,8940,8950 +1790784842.045244238,8940,8950 +1790784842.151786109,8940,8950 +1790784842.263704781,8940,8950 +1790784842.369130009,8940,8950 +1790784842.477903785,8940,8950 +1790784842.588632003,8940,8950 +1790784842.699574794,8940,8950 +1790784842.807851905,8940,8950 +1790784842.914501986,8940,8950 +1790784843.024972422,8940,8950 +1790784843.132997640,8940,8950 +1790784843.240550022,8940,8950 +1790784843.351505592,8940,8950 +1790784843.460716013,8940,8950 +1790784843.565157060,8940,8950 +1790784843.670607349,8940,8950 +1790784843.777910668,8940,8950 +1790784843.885650132,8940,8950 +1790784843.992623438,8940,8950 +1790784844.102109352,8940,8950 +1790784844.206029613,8940,8950 +1790784844.315593998,8940,8950 +1790784844.423438393,8940,8950 +1790784844.532658474,8940,8950 +1790784844.644083341,8940,8950 +1790784844.756166455,8976,8986 +1790784844.863378873,8976,8986 +1790784844.969615260,8976,8986 +1790784845.077176452,8976,8986 +1790784845.186646737,8976,8986 +1790784845.297756869,8976,8986 +1790784845.405269981,8976,8986 +1790784845.509978971,8976,8986 +1790784845.616368460,8976,8986 +1790784845.728506094,8976,8986 +1790784845.837024557,8976,8986 +1790784845.944602600,8976,8986 +1790784846.053819551,8976,8986 +1790784846.158147666,8976,8986 +1790784846.269527622,8976,8986 +1790784846.377197246,8976,8986 +1790784846.486284535,8976,8986 +1790784846.595597518,8976,8986 +1790784846.701476691,8976,8986 +1790784846.808092872,8976,8986 +1790784846.918805310,8976,8986 +1790784847.028258735,8976,8986 +1790784847.137180722,8976,8986 +1790784847.247341394,8976,8986 +1790784847.351956253,8976,8986 +1790784847.461380336,8976,8986 +1790784847.570969502,8976,8986 +1790784847.679181311,8976,8986 +1790784847.789767948,8976,8986 +1790784847.900785989,8976,8986 +1790784848.007854036,8976,8986 +1790784848.117272920,8976,8986 +1790784848.227755005,8976,8986 +1790784848.336938856,8976,8986 +1790784848.445953755,8976,8986 +1790784848.551329822,8976,8986 +1790784848.655110751,8976,8986 +1790784848.763906808,8976,8986 +1790784848.871537271,8976,8986 +1790784848.982150578,8976,8986 +1790784849.093308071,8976,8986 +1790784849.202139028,8976,8986 +1790784849.311448820,8976,8986 +1790784849.417937740,8976,8986 +1790784849.526239621,8976,8986 +1790784849.635773666,8976,8986 +1790784849.744765174,8976,8986 +1790784849.853818473,8976,8986 +1790784849.968579378,8976,9194 +1790784850.083119509,9184,9450 +1790784850.191399150,9440,9450 +1790784850.306316226,9440,9450 +1790784850.411863855,9440,9450 +1790784850.517646597,9440,9450 +1790784850.632442351,9440,9450 +1790784850.738746840,9440,9450 +1790784850.842557339,9440,9450 +1790784850.956718527,9440,9450 +1790784851.064014386,9440,9450 +1790784851.168078909,9440,9450 +1790784851.274390766,9440,9450 +1790784851.377619040,9440,9450 +1790784851.480258046,9440,9450 +1790784851.578804946,9440,9450 +1790784851.688752485,9440,9452 +1790784851.800234642,9442,9452 +1790784851.914939636,9442,9452 +1790784852.021917041,9442,9452 +1790784852.134913436,9442,9452 +1790784852.240473405,9442,9452 +1790784852.354840755,9442,9452 +1790784852.461851430,9442,9452 +1790784852.573556660,9442,9452 +1790784852.680799839,9442,9452 +1790784852.786910894,9442,9452 +1790784852.892240601,9442,9452 +1790784853.012264206,9442,9452 +1790784853.122448928,9442,9452 +1790784853.235198039,9442,9452 +1790784853.347670017,9442,9452 +1790784853.449713626,9442,9452 +1790784853.567508975,9442,9452 +1790784853.672790711,9442,9452 +1790784853.786583585,9442,9452 +1790784853.889266562,9442,9452 +1790784854.000905370,9442,9452 +1790784854.112077814,9442,9452 +1790784854.226280591,9442,9452 +1790784854.333266277,9442,9452 +1790784854.443139746,9442,9452 +1790784854.560525139,9442,9452 +1790784854.663936945,9442,9452 +1790784854.779531578,9442,9452 +1790784854.885843556,9442,9452 +1790784855.000842533,9442,9452 +1790784855.111911455,9442,9452 +1790784855.224620386,9442,9452 +1790784855.342765179,9442,9452 +1790784855.450572984,9442,9452 +1790784855.561338922,9442,9452 +1790784855.672750839,9442,9452 +1790784855.780984238,9442,9452 +1790784855.899126611,9442,9452 +1790784856.007095178,9442,9452 +1790784856.113366955,9442,9632 +1790784856.219424541,9622,9632 +1790784856.329464221,9298,9632 +1790784856.435051641,9622,9632 +1790784856.544437934,9086,9632 +1790784856.653285481,9622,9632 +1790784856.765000731,9622,9632 +1790784856.876619439,9622,9352 +1790784856.985296994,9622,9632 +1790784857.094664617,9622,9632 +1790784857.203255161,9622,9632 +1790784857.310807983,8846,9632 +1790784857.421135687,9622,9632 +1790784857.532067398,9622,9632 +1790784857.640677232,9622,8832 +1790784857.752193719,9622,9632 +1790784857.857883530,9622,9632 +1790784857.967985751,9622,9632 +1790784858.077865250,9054,9632 +1790784858.187279633,9622,9632 +1790784858.296673487,9622,9632 +1790784858.403095896,9622,8832 +1790784858.511037952,9622,9632 +1790784858.617774445,9622,9632 +1790784858.724422457,9622,9632 +1790784858.832626887,8822,9632 +1790784858.944241705,9622,9632 +1790784859.053527337,9622,9632 +1790784859.157472018,9622,8832 +1790784859.269198838,9622,9632 +1790784859.375705508,9622,9632 +1790784859.483808867,9622,9632 +1790784859.591643532,8822,9632 +1790784859.702018216,9622,9632 +1790784859.813037438,9622,9632 +1790784859.921380409,9622,8832 +1790784860.029826422,9622,9632 +1790784860.138965312,9622,9632 +1790784860.248713139,9622,9632 +1790784860.357747208,8886,9632 +1790784860.465483352,9622,9632 +1790784860.575323270,9622,9632 +1790784860.683098435,9622,8832 +1790784860.792802142,9622,9632 +1790784860.901529227,9622,9632 +1790784861.009224511,9622,9632 +1790784861.119577645,8842,9632 +1790784861.229036669,9622,9632 +1790784861.338048338,9622,9632 +1790784861.444839331,9622,8832 +1790784861.551814167,9622,9632 +1790784861.663879340,9622,9632 +1790784861.778786706,9622,9632 +1790784861.887691954,8846,9632 +1790784861.995777822,9622,9632 +1790784862.108031298,9622,9632 +1790784862.217377351,9622,8832 +1790784862.327212869,9622,9632 +1790784862.433972042,9622,9632 +1790784862.543170903,9622,9632 +1790784862.652600006,8886,9632 +1790784862.762923930,9622,9632 +1790784862.873807410,9622,9632 +1790784862.977936743,9622,8832 +1790784863.083421062,9622,9632 +1790784863.193304061,9622,9632 +1790784863.302055537,9622,9632 +1790784863.409692520,8822,9632 +1790784863.518970892,9622,9632 +1790784863.628263733,9622,9632 +1790784863.737467625,9622,8832 +1790784863.843559210,9622,9632 +1790784863.955210129,9622,9632 +1790784864.065154058,9622,9632 +1790784864.171474986,8842,9632 +1790784864.281174563,9622,9632 +1790784864.392411607,9622,9632 +1790784864.497381960,9622,8832 +1790784864.608629464,9622,9632 +1790784864.721489917,9622,9632 +1790784864.830064060,9622,9632 +1790784864.938623124,8846,9632 +1790784865.047224179,9622,9632 +1790784865.157909776,9622,9632 +1790784865.264820671,9622,8832 +1790784865.371432952,9622,9632 +1790784865.480347880,9622,9632 +1790784865.587256855,9622,9632 +1790784865.693656794,8822,9632 +1790784865.799111113,9622,9632 +1790784865.905782825,9622,9632 +1790784866.013428418,9622,9632 +1790784866.121115482,9622,9632 +1790784866.229950018,9622,9632 +1790784866.337133197,9622,9632 +1790784866.447401559,9622,9632 +1790784866.556121655,9622,9632 +1790784866.669620695,9086,9632 +1790784866.776811143,9622,9632 +1790784866.884279354,9622,9632 +1790784866.993432144,9622,8832 +1790784867.100343910,9622,9632 +1790784867.210952687,9622,9632 +1790784867.321411782,9622,9632 +1790784867.423813486,8822,9632 +1790784867.532517251,9622,9632 +1790784867.640282685,9622,9632 +1790784867.748018679,9622,8832 +1790784867.853301246,9158,9168 +1790784867.965229298,9414,9424 +1790784868.067654142,9414,9424 +1790784868.180792357,9414,9424 +1790784868.284282814,9414,9424 +1790784868.394907771,9414,9424 +1790784868.504053821,9414,9424 +1790784868.610642072,9414,9424 +1790784868.714758446,9414,9424 +1790784868.828753581,9414,9424 +1790784868.930115303,9414,9424 +1790784869.033250095,9414,9424 +1790784869.135279164,9414,9424 +1790784869.240684412,9414,9424 +1790784869.347988461,9414,9424 +1790784869.453828324,9414,9424 +1790784869.567783439,9414,9424 +1790784869.674403201,9414,9424 +1790784869.790036275,9414,9424 +1790784869.898283765,9414,9424 +1790784870.011201938,9414,9424 +1790784870.118969493,9414,9424 +1790784870.231591943,9414,9424 +1790784870.336509765,9414,9424 +1790784870.451617973,9414,9424 +1790784870.560329178,9414,9424 +1790784870.667447295,9414,9424 +1790784870.770769829,9414,9424 +1790784870.885837997,9414,9424 +1790784870.997798749,9414,9424 +1790784871.109033914,9414,9424 +1790784871.228031936,9414,9424 +1790784871.339123868,9414,9424 +1790784871.452856121,9414,9424 +1790784871.561802299,9414,9424 +1790784871.674893604,9414,9424 +1790784871.777392499,9414,9424 +1790784871.889179179,9414,9424 +1790784871.993896449,9414,9424 +1790784872.107388649,9414,9424 +1790784872.213488254,9414,9424 +1790784872.327455439,9414,9424 +1790784872.439858947,9414,9424 +1790784872.547589241,9414,9424 +1790784872.661579976,9414,9424 +1790784872.761745945,9414,9424 +1790784872.875366456,9414,9424 +1790784872.978879902,9414,9424 +1790784873.094528276,9414,9424 +1790784873.205617119,9414,9424 +1790784873.316018573,9414,9424 +1790784873.426057194,9414,9424 +1790784873.529992205,9414,9424 +1790784873.645991613,9414,9424 +1790784873.755778741,9414,9424 +1790784873.870173441,9414,9498 +1790784873.976838562,9488,9498 +1790784874.084382605,9274,9740 +1790784874.194119542,9730,9740 +1790784874.304554907,9490,9740 +1790784874.412089459,9730,9740 +1790784874.523181871,9670,9740 +1790784874.632020348,9730,9740 +1790784874.737669349,9730,9740 +1790784874.847566828,9730,9740 +1790784874.958848382,9730,9740 +1790784875.068351206,9730,9740 +1790784875.178746371,9730,9740 +1790784875.293592706,9730,9740 +1790784875.400238458,9706,9740 +1790784875.509157785,9730,9740 +1790784875.617391796,9730,9740 +1790784875.725715537,9730,9740 +1790784875.835957439,9730,9740 +1790784875.943829525,9730,9740 +1790784876.053817885,9730,9740 +1790784876.161529939,9730,9740 +1790784876.272291578,9730,9740 +1790784876.381951674,9730,9740 +1790784876.491350547,9730,9740 +1790784876.596781305,9730,9740 +1790784876.709380745,9730,9740 +1790784876.820900662,9730,9740 +1790784876.933796415,9670,9740 +1790784877.043391450,9730,9740 +1790784877.155275292,9730,9740 +1790784877.261576930,9730,9740 +1790784877.371546620,9730,9740 +1790784877.480177914,9730,9740 +1790784877.591435868,9730,9740 +1790784877.705106060,9730,9740 +1790784877.815468484,9706,9740 +1790784877.925702177,9730,9740 +1790784878.037256775,9730,9740 +1790784878.145753067,9730,9740 +1790784878.256211153,9730,9740 +1790784878.365974240,9730,9740 +1790784878.473628743,9730,9740 +1790784878.581264537,9730,9740 +1790784878.691358558,9730,9740 +1790784878.799099322,9730,9740 +1790784878.904329378,9730,9740 +1790784879.015165637,9730,9740 +1790784879.124404649,9730,9740 +1790784879.233778612,9730,9740 +1790784879.341117691,9730,9740 +1790784879.450610166,9730,9740 +1790784879.560270131,9730,9740 +1790784879.669657805,9730,9740 +1790784879.780577655,9730,9740 +1790784879.891221903,9730,9740 +1790784879.999008097,9730,9740 +1790784880.105889622,9730,9740 +1790784880.217646952,9730,9740 +1790784880.326121495,9730,9740 +1790784880.433967880,9730,9740 +1790784880.542030328,9730,9740 +1790784880.649006824,9730,9740 +1790784880.759103315,9730,9740 +1790784880.860934092,9730,9740 +1790784880.971905443,9730,9740 +1790784881.081160765,9730,9740 +1790784881.189650628,9730,9740 +1790784881.300626039,9730,9740 +1790784881.409310004,9730,9740 +1790784881.521377227,9730,9740 +1790784881.631581520,9730,9740 +1790784881.743579083,9730,9740 +1790784881.851274566,9730,9740 +1790784881.962518060,9730,9740 +1790784882.066421361,9730,9740 +1790784882.175248138,9730,9740 +1790784882.287239931,9730,9740 +1790784882.397215320,9730,9740 +1790784882.506669734,9730,9740 +1790784882.617745186,9730,9740 +1790784882.728842089,9730,9740 +1790784882.837927708,9730,9740 +1790784882.948230832,9730,9740 +1790784883.056474252,9730,9740 +1790784883.164252647,9730,9740 +1790784883.277072549,9730,9740 +1790784883.387723176,9730,9740 +1790784883.498067130,9730,9740 +1790784883.612234867,9730,9740 +1790784883.723227479,9490,9740 +1790784883.835817918,9730,9740 +1790784883.949109736,9454,9740 +1790784884.058970014,9730,9740 +1790784884.171426662,9598,9740 +1790784884.282096240,9730,9740 +1790784884.391281910,9706,9740 +1790784884.506622511,9730,9740 +1790784884.619748397,9490,9740 +1790784884.728836776,9730,9740 +1790784884.841038491,9408,9740 +1790784884.949748087,9730,9740 +1790784885.057521241,9730,9740 +1790784885.163725958,9730,9740 +1790784885.272325902,9730,9740 +1790784885.381639335,9730,9740 +1790784885.492654876,9730,9740 +1790784885.601467612,9730,9740 +1790784885.710286839,9730,9740 +1790784885.817714430,9730,9740 +1790784885.923193569,9730,9740 +1790784886.033307930,9730,9740 +1790784886.145146401,9730,9740 +1790784886.258144435,9730,9740 +1790784886.370981047,9706,9740 +1790784886.476054452,9730,9740 +1790784886.590099487,9730,9740 +1790784886.701840737,9730,9740 +1790784886.813828540,9598,9740 +1790784886.920488022,9730,9740 +1790784887.031444313,9730,9740 +1790784887.144189484,9730,9740 +1790784887.255115114,9526,9740 +1790784887.363396545,9730,9740 +1790784887.469661002,9730,8832 +1790784887.577506708,8828,8838 +1790784887.690284259,9202,9212 +1790784887.801389572,9202,9212 +1790784887.911246011,9202,9212 +1790784888.020753355,9202,10054 +1790784888.126142413,10044,10054 +1790784888.236906711,10044,9748 +1790784888.348918684,9738,9748 +1790784888.461375382,9714,9530 +1790784888.573342955,9766,9776 +1790784888.684919743,9766,9784 +1790784888.794602849,9774,9784 +1790784888.897875983,9774,9812 +1790784889.006033031,9802,9812 +1790784889.117258976,9802,9820 +1790784889.230000197,9810,9820 +1790784889.347059997,9810,9848 +1790784889.457350800,9838,9848 +1790784889.566911055,9598,9774 +1790784889.670886057,9846,9856 +1790784889.783860871,9846,9884 +1790784889.892575616,9874,9884 +1790784890.000939747,9874,9574 +1790784890.112767538,9882,9892 +1790784890.217654590,9882,8984 +1790784890.331425903,9910,9920 +1790784890.440848776,9910,9928 +1790784890.557042757,9918,9928 +1790784890.668182450,9918,9956 +1790784890.776027535,9946,9956 +1790784890.882118941,9946,9428 +1790784890.991947769,9954,9964 +1790784891.100844867,9954,9056 +1790784891.212885120,9984,9994 +1790784891.323020501,9984,9920 +1790784891.429865606,9992,10002 +1790784891.539394740,9992,9094 +1790784891.647451098,10020,10030 +1790784891.754996100,10020,9122 +1790784891.863958238,10028,10038 +1790784891.972383181,10028,9382 +1790784892.083919678,10056,10066 +1790784892.190639521,10056,10066 +1790784892.300826663,10064,10074 +1790784892.409316026,10064,10074 +1790784892.520015124,9256,10102 +1790784892.627437514,10092,10102 +1790784892.737883920,9218,10110 +1790784892.848022301,10100,10110 +1790784892.951846011,9192,10138 +1790784893.063021094,10128,10138 +1790784893.172892043,9220,10146 +1790784893.283185016,10136,10146 +1790784893.396056559,9932,10174 +1790784893.509288126,10164,10174 +1790784893.622345880,10164,10182 +1790784893.733334182,10172,10182 +1790784893.840256337,10172,9892 +1790784893.948281305,10200,10210 +1790784894.061788504,10200,9302 +1790784894.170377618,10208,10218 +1790784894.276619085,10208,9310 +1790784894.386158330,10236,10246 +1790784894.494196668,10236,9344 +1790784894.601866151,10244,10254 +1790784894.711748280,10244,10254 +1790784894.824526342,10274,10284 +1790784894.935219980,10274,10284 +1790784895.043491660,9820,10292 +1790784895.150242083,10282,10292 +1790784895.255033464,9374,9384 +1790784895.361892008,9922,10188 +1790784895.474161884,10178,10188 +1790784895.584167304,10178,10188 +1790784895.699548555,10178,10188 +1790784895.808024808,10178,10188 +1790784895.929071603,10178,10188 +1790784896.037055070,10178,10188 +1790784896.141514268,10178,10188 +1790784896.253346519,10178,10188 +1790784896.368985533,10178,10188 +1790784896.477089371,10178,10188 +1790784896.583855694,10178,10188 +1790784896.688464803,10178,10188 +1790784896.793249374,10178,10188 +1790784896.896082922,10178,10188 +1790784897.001941156,10178,10188 +1790784897.117501429,10178,10188 +1790784897.231678496,10178,10188 +1790784897.340598044,10178,10188 +1790784897.461455647,10178,10188 +1790784897.566260818,10178,10188 +1790784897.685966929,10178,10188 +1790784897.795095829,10178,10188 +1790784897.903258848,10178,10188 +1790784898.017888030,10178,10188 +1790784898.122620841,10178,10188 +1790784898.225908464,10178,10188 +1790784898.344258220,10178,10188 +1790784898.450581108,10178,10188 +1790784898.559656377,10178,10188 +1790784898.672679181,10178,10188 +1790784898.779569346,10178,10188 +1790784898.896972600,10178,10188 +1790784898.999866320,10178,10188 +1790784899.117273464,10178,10188 +1790784899.223795974,10178,10188 +1790784899.335671896,10178,10188 +1790784899.449055794,10178,10188 +1790784899.562315061,10178,10188 +1790784899.669729042,10178,10188 +1790784899.781893846,10178,10188 +1790784899.887445256,10178,10188 +1790784899.999847773,10178,10188 +1790784900.112349361,10178,10188 +1790784900.224877210,10178,10188 +1790784900.343493348,10178,10188 +1790784900.452116853,10178,10188 +1790784900.565319019,10178,10188 +1790784900.677655546,10178,10188 +1790784900.796619918,10178,10188 +1790784900.917230988,10178,10188 +1790784901.027718374,10178,10188 +1790784901.143432758,10178,10188 +1790784901.255085267,10178,10188 +1790784901.367885039,10180,10356 +1790784901.473237336,10346,10356 +1790784901.579276041,10346,10146 +1790784901.691890761,9992,10382 +1790784901.804997717,10372,10382 +1790784901.913834713,10372,9384 +1790784902.019840428,10370,10380 +1790784902.129484164,10370,9818 +1790784902.233031150,10336,10346 +1790784902.341560194,10336,10014 +1790784902.452917520,10382,10392 +1790784902.563067761,10382,10392 +1790784902.669841064,10382,9732 +1790784902.776014621,10336,10346 +1790784902.885546576,10336,10232 +1790784902.993360831,10382,10392 +1790784903.104108339,10382,10392 +1790784903.213481212,10382,10104 +1790784903.319723709,10336,10346 +1790784903.429912432,10336,10394 +1790784903.541423549,10382,10392 +1790784903.652096157,10382,10392 +1790784903.763688134,10382,10320 +1790784903.868329424,10336,10346 +1790784903.979223514,10336,10232 +1790784904.087971990,10382,10392 +1790784904.201092645,10382,10392 +1790784904.313361721,10382,10314 +1790784904.418804039,10382,10392 +1790784904.527098690,10382,10392 +1790784904.634129396,10382,9384 +1790784904.740546596,10336,10346 +1790784904.847993757,10336,10048 +1790784904.959611855,10382,10392 +1790784905.067641702,10382,10392 +1790784905.172104930,10382,9384 +1790784905.279912985,9374,10232 +1790784905.381876274,10344,10354 +1790784905.483506918,10344,9384 +1790784905.585771040,10344,10354 +1790784905.691650804,10344,9384 +1790784905.797342544,10350,10360 +1790784905.905142640,10350,9424 +1790784906.007740026,10350,10360 +1790784906.113063813,10350,10360 +1790784906.219076117,10350,10360 +1790784906.325234794,9374,9384 +1790784906.426721137,10350,10360 +1790784906.529097660,10350,9384 +1790784906.629960156,10350,10360 +1790784906.730698561,10350,9384 +1790784906.835508312,10350,10360 +1790784906.936047225,10350,9384 +1790784907.038060004,10350,10360 +1790784907.139970332,10350,9586 +1790784907.244046785,10350,10360 +1790784907.349415192,10350,10360 +1790784907.449751173,10350,10360 +1790784907.557198523,10310,9410 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/checks.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/checks.json new file mode 100644 index 0000000..48068a6 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/checks.json @@ -0,0 +1,488 @@ +{ + "url": "http://127.0.0.1:18033", + "started_utc": "2026-09-30T16:24:52Z", + "health": { + "status": "ok", + "model": { + "name": "Intern-Decision-4B", + "source": "internlm/Intern-Decision-4B", + "revision": "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "checkpoint": "/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", + "inference_py_sha256": "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863", + "temperature": 1.99241824, + "dtype": "bfloat16", + "attn_implementation": "sdpa", + "device": "cuda", + "max_length": 8192, + "torch_version": "2.10.0+cu128", + "transformers_version": "5.17.0", + "vision_tower": "removed", + "device_name": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition", + "allocated_gib": 7.937, + "reserved_gib": 7.969, + "max_reserved_gib": 8.861 + }, + "vram_cap_gib": 9.25, + "max_tokens": 8192, + "max_decisions": 64, + "max_questions_per_call": 16, + "chunking": "/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.", + "workloads": [] + }, + "auth": { + "/decide": { + "no_token": 401, + "wrong_token": 401, + "right_token": 200 + }, + "/decide/shared": { + "no_token": 401, + "wrong_token": 401, + "right_token": 200 + }, + "health_no_token": 200, + "pass": true, + "t_start": 1790785492.2528422, + "t_end": 1790785492.3375697 + }, + "chunking": { + "status": [ + 200, + 200, + 200 + ], + "timing": { + "total_seconds": 0.15066236606799066, + "batch_size": 20, + "calls": 2, + "questions_per_call": [ + 16, + 4 + ], + "input_tokens": [ + 1884, + 1328 + ], + "inference_seconds": 0.13768 + }, + "rows": [ + { + "id": "d0", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q1", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d1", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q2", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d2", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q3", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d3", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q4", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d4", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q5", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d5", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q6", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d6", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q7", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d7", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q8", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d8", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q9", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d9", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q10", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d10", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q11", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d11", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q12", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d12", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q13", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d13", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q14", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d14", + "top_whole": "no", + "top_split": "no", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q15", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d15", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 0, + "field": "q16", + "questions": 16 + }, + "sha_equal": true + }, + { + "id": "d16", + "top_whole": "unclear", + "top_split": "unclear", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q1", + "questions": 4 + }, + "sha_equal": true + }, + { + "id": "d17", + "top_whole": "unclear", + "top_split": "unclear", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q2", + "questions": 4 + }, + "sha_equal": true + }, + { + "id": "d18", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q3", + "questions": 4 + }, + "sha_equal": true + }, + { + "id": "d19", + "top_whole": "yes", + "top_split": "yes", + "max_dp": 0.0, + "call_whole": { + "index": 1, + "field": "q4", + "questions": 4 + }, + "sha_equal": true + } + ], + "e2e_ms": 157.3, + "pass": true, + "t_start": 1790785492.3376284, + "t_end": 1790785492.6581485 + }, + "queue": { + "sent": 48, + "max_queue": 32, + "ok": 32, + "busy_429": 16, + "codes_429": [ + "busy" + ], + "other": [], + "pass": true, + "t_start": 1790785492.6582093, + "t_end": 1790785499.6040177 + }, + "fit": { + "largest_ok_state_chars": 9508, + "largest_ok_tokens": 8191, + "probes": [ + [ + 31950, + 422, + null + ], + [ + 16475, + 422, + null + ], + [ + 8737, + 200, + 8005 + ], + [ + 12606, + 422, + null + ], + [ + 10671, + 422, + null + ], + [ + 9704, + 422, + null + ], + [ + 9220, + 200, + 8120 + ], + [ + 9462, + 200, + 8176 + ], + [ + 9583, + 422, + null + ], + [ + 9522, + 422, + null + ], + [ + 9492, + 200, + 8185 + ], + [ + 9507, + 200, + 8191 + ], + [ + 9514, + 422, + null + ], + [ + 9510, + 422, + null + ], + [ + 9508, + 200, + 8191 + ], + [ + 9509, + 422, + null + ] + ], + "t_start": 1790785499.6040773, + "t_end": 1790785508.5767787 + }, + "maxreq": { + "state_chars": 9508, + "tokens_per_call": 8191, + "decisions": 64, + "options_each": 16, + "body_bytes": 101019, + "runs": [ + { + "status": 200, + "e2e_ms": 1671.9, + "code": null, + "calls": 4, + "input_tokens": [ + 8191, + 8191, + 8191, + 8191 + ], + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.25, + "t_end": 1790785520.5513089 + }, + { + "status": 200, + "e2e_ms": 1672.4, + "code": null, + "calls": 4, + "input_tokens": [ + 8191, + 8191, + 8191, + 8191 + ], + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.25, + "t_end": 1790785522.2301269 + }, + { + "status": 200, + "e2e_ms": 1675.4, + "code": null, + "calls": 4, + "input_tokens": [ + 8191, + 8191, + 8191, + 8191 + ], + "reserved_gib_after": 7.969, + "max_reserved_gib": 9.25, + "t_end": 1790785523.9114661 + } + ], + "t_start": 1790785508.5768383, + "t_end": 1790785523.9116056 + }, + "oom": { + "oom_status": 200, + "oom_error": null, + "oom_ms": 1665.4, + "reserved_gib": { + "before": 7.969, + "after_oom": 7.969, + "after_next_request": 7.975 + }, + "next_request_status": 200, + "next_top": "yes", + "pass": false, + "t_start": 1790785523.9116578, + "t_end": 1790785525.6274366 + }, + "finished_utc": "2026-09-30T16:25:25Z" +} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/container.id b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/container.id new file mode 100644 index 0000000..bfe2aa3 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/container.id @@ -0,0 +1 @@ +5068d9162b114ffd3b9773ce3a001634a40605e8e9777778e586bc28475e2c9f diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/events.txt b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/events.txt new file mode 100644 index 0000000..6663ec0 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/events.txt @@ -0,0 +1,5 @@ +1790785352.925568640 sets:start +1790785446.620798652 sets:done +1790785492.200303780 shape:done +1790785525.645862796 checks:done +gpu3 after: 2 MiB diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/health-end.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/health-end.json new file mode 100644 index 0000000..bb1f747 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/health-end.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.975,"max_reserved_gib":9.25},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/health-rest.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/health-rest.json new file mode 100644 index 0000000..37aac99 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/health-rest.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.969,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/sets.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/sets.json new file mode 100644 index 0000000..53e4050 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/sets.json @@ -0,0 +1 @@ +{"label": "r1", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:22:32Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94820002280176, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 37.79}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.535361014306545, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.71}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17208607122302, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.38}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.195066921412945, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.54}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.009123945608735, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.59}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9531640317291, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.39}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12933603674173, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.27}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64115204103291, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.87}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58631111122668, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.67}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.55531798116863, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 39.72}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24834695458412, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.48}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.22773904539645, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 35.31}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47352911159396, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.62}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89836384728551, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.08}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.053666008636355, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 34.47}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.318547954782844, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.41}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88332298770547, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.21}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08775597624481, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.39}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92354308441281, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.19}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.221837090328336, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.45}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85288317501545, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.14}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65892989374697, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 33.96}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9726549834013, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.16}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29426790960133, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.44}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.111615063622594, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.29}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14233706705272, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.3}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.398729007691145, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.45}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.843542009592056, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.18}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96078388020396, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.1}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.239377154037356, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 34.42}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.572568817064166, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 33.89}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31391881592572, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.53}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.897932877764106, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 33.98}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24824800156057, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.52}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79928299970925, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.12}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.445679074153304, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.59}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60768014378846, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 33.97}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93267400376499, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.23}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72629198990762, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 33.99}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92372399382293, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.05}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80329187773168, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.03}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75046213902533, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.07}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.474708933383226, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 33.78}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12360584177077, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.29}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.624329863116145, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 33.89}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96562419645488, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.25}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.743010859936476, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.01}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0652349665761, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.26}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.786722019314766, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.15}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02150494791567, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.28}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93306492641568, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.09}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.946953972801566, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.24}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75980190746486, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.03}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.049975944682956, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.42}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68469004333019, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 33.95}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.205856991931796, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.25}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02237503603101, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.23}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90913296304643, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.13}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1175860054791, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.17}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17133612371981, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.34}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93176410160959, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.29}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.956154042854905, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.25}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79595189169049, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.03}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7596719879657, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.15}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63025121204555, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 33.98}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80672193877399, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.09}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75284190103412, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 33.98}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13071602396667, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.31}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.569199059158564, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 33.89}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11119596846402, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.41}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85413300991058, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.03}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98045411147177, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.23}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08484605886042, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.21}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21687709912658, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.33}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79137188009918, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.0}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.060525035485625, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.42}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94747411645949, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 34.4}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.04089194536209, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 39.48}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.013485096395016, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.36}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12630597874522, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.42}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85632301494479, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.14}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.790671991184354, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.16}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.675481125712395, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 33.93}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52334089949727, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.55}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28306805714965, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.47}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.945984000340104, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.25}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48635791614652, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 33.75}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17310703359544, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.42}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08837600424886, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.45}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.075054831802845, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.36}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77427303045988, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 33.84}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10189484991133, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 34.38}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.722911056131124, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.01}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80957295000553, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.07}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85495210811496, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.11}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85369319282472, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.24}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73954214900732, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.12}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05920488573611, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.37}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.880323896184564, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.15}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87339392490685, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.24}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.801562178879976, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.15}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04755613580346, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.23}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90757299773395, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.16}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93301300518215, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.29}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80754313245416, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.1}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.369557889178395, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.61}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.701582139357924, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 33.9}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.562331184744835, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.64}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97324404492974, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.09}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21895697526634, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.33}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79869300685823, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.08}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07209601998329, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.25}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8491830304265, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.07}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13976684957743, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.41}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86474310234189, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.14}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.932253977283835, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.25}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.834912141785026, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.08}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82521195150912, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.18}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76381194964051, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 33.98}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3505090828985, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.41}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60912998020649, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 33.91}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.919774021953344, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.23}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92990285344422, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.01}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0940959546715, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.3}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.63965011946857, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 33.96}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10892586968839, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.45}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.962704034522176, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 33.96}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.179487058892846, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.32}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.829713033512235, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.0}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2511581517756, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.38}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05820511095226, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.08}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.158936029300094, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.37}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20136685296893, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.38}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20108699053526, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.32}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01734519563615, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.25}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.249717861413956, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.4}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11690497584641, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.17}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.179416976869106, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.42}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.897504003718495, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.06}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.410537945106626, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.45}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.988025065511465, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.12}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06623497419059, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.2}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9784040376544, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.08}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13447693735361, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.26}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13539591990411, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.25}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.135358994826674, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 36.19}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3999688308686, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.55}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21316694281995, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.26}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95327299647033, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.18}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35593915916979, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 34.52}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03328501433134, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 34.07}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.044194992631674, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.26}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.048676051199436, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 34.19}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30281791649759, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.34}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.853202851489186, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.05}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39230893552303, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 34.49}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.357109133154154, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.41}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01881505548954, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.25}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7967130150646, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 33.97}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49303985387087, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.59}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98196494951844, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 34.05}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.550101056694984, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 34.71}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04217518679798, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 34.1}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17034589499235, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 34.4}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18350687995553, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 34.33}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.060424918308854, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.24}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.726071033626795, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 33.91}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.316217785701156, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.45}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05377480946481, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.09}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21836721524596, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 34.29}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00398514047265, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 34.2}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32686792500317, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.47}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81548195891082, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 34.07}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09383588284254, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 34.43}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88553278334439, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 33.96}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.115016020834446, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.35}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6907708812505, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 33.92}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73007106408477, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 34.07}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84374317526817, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 33.95}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.130576092749834, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 34.27}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.795332096517086, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 33.98}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92095400765538, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 34.17}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.955303978174925, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 33.93}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30505798012018, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.44}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00483497232199, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 34.11}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.029133876785636, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 34.17}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28641802445054, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.34}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20327699556947, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 34.42}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.010753992944956, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 34.12}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19030716456473, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.4}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.688210908323526, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 33.96}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04115515574813, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.21}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88024287112057, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 33.86}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24771784618497, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 34.39}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.953563103452325, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 34.09}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31153812259436, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.37}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00296487659216, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 34.07}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.720912136137486, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 34.92}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72672202810645, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 33.86}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34420915134251, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 34.59}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.833371967077255, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.05}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34205896034837, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.37}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.088614888489246, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 34.24}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.03536598570645, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 35.13}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83567279949784, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.96}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4649700243026, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.65}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65111097507179, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 33.92}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.243987899273634, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 34.44}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.761600989848375, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 33.92}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.27996391616762, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 38.2}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59283309429884, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 35.59}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34103892929852, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 34.51}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.325127981603146, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 34.52}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61847107671201, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 34.7}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.882343934848905, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 34.02}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68717311322689, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 34.77}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89625416882336, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.11}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45193011127412, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 34.5}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.390118930488825, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 34.44}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.273587891831994, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.49}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84685309417546, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 33.98}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79924311675131, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 34.85}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85040306299925, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.12}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15829714201391, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.24}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46168897487223, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 34.51}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.192136980593204, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.35}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06578491255641, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 34.1}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.766392808407545, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 34.89}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99325514025986, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 34.08}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55984082818031, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.74}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.138886051252484, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 34.32}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24196809343994, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.41}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14260598644614, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 34.2}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.382748909294605, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 34.59}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98338405229151, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 34.12}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46628016233444, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 34.55}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97164403274655, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 34.18}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57069096900523, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 34.72}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86872310936451, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 34.05}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51317993737757, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 34.59}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.147586001083255, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 34.2}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25199704058468, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 34.34}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02527494169772, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 34.19}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.700602086260915, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.68}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94109292700887, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 34.02}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68533212132752, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.78}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19781688414514, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.24}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.228906992822886, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 34.36}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46028896421194, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 34.55}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.109936121851206, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.27}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93357389420271, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 34.09}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.675981875509024, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.79}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.154488895088434, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 35.31}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38217893987894, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.55}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27827803231776, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 34.37}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99026512913406, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.17}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36469894647598, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 34.55}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0768450666219, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.12}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00110502541065, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30787802487612, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.36}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17252588830888, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34379797242582, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.57}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36560791730881, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.742941942065954, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.0}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.155695958063006, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.48}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65080992504954, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.01}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.272477988153696, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 34.47}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96613386273384, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08306513726711, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27137786336243, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.48}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20650705508888, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 34.39}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85373207554221, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.15}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.456130143254995, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 34.53}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.82514099031687, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 39.94}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56657103076577, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 34.63}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21437696367502, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.54}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.172317039221525, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51212009228766, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.67}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22497681155801, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.943993996828794, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.06}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27696789428592, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 34.67}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94173414260149, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.27}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48416109010577, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.514761090278625, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.62}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48220996931195, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 34.61}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.410780088976026, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.504239939153194, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 34.72}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.957573844119906, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.17}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61398186907172, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 34.67}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1811268851161, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.39}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22405596636236, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.645821925252676, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.77}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.295887945219874, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 34.43}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5420399941504, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.68}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36219904385507, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 34.54}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99721395969391, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.19}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61790180578828, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 34.76}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.009275052696466, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.19}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46795002371073, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 34.64}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.371008889749646, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.54}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44784905575216, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 34.52}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26259805262089, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.47}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24372712895274, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06726594828069, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.18}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.515830947086215, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 34.73}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1795161627233, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5894820317626, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71967301145196, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.74}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23466699011624, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.816091975197196, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.18}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10089600645006, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 34.37}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.883443128317595, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.734583020210266, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 34.93}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13606693781912, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.33}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18369617126882, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 34.53}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27620793506503, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.61}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.230366840958595, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83876199647784, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.15}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15554694645107, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9939049705863, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.552200024947524, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 34.74}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71164291538298, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.71}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.078156135976315, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55164099857211, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.71}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69890109449625, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.856313003227115, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.26}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83758294209838, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 34.02}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.974994000047445, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.19}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27426682598889, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 34.28}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94904409348965, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.17}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88144311308861, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.392138969153166, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89838293939829, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.951873917132616, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.17}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38778899423778, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 34.54}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03446500003338, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13087597861886, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 34.22}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10695612244308, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.21}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96955391205847, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 34.07}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56254189647734, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.71}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04844508320093, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 34.14}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.531932048499584, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.28}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.797253811731935, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.987744038924575, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.24}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00550482608378, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 34.2}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.103955866768956, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.18}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.886021960526705, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2942581307143, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.45}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.146695889532566, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.108306074514985, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.22}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28280798532069, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.45}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.977904848754406, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.13}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00144519098103, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 34.09}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.117896135896444, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10838593542576, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 34.21}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.068676203489304, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.28}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86629398725927, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.098916014656425, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.24}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.608930841088295, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 34.77}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02815482392907, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.21}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.915672942996025, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 34.1}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36546915024519, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.47}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71904210932553, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17365697957575, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.41}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87266283668578, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0923459995538, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.18}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.304427940398455, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 34.49}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15336602181196, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.18}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.222126964479685, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 34.23}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19422593712807, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.35}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79002192802727, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 33.95}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22409694455564, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.38}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95443389005959, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 34.21}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.160035921260715, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.33}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91783384419978, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 34.01}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68736107647419, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72786196693778, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 33.99}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90601303242147, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.24}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00278513133526, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 34.22}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.897352896630764, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.1}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92484297789633, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92432399839163, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.24}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.657300079241395, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 33.85}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.939363926649094, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.17}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81180300191045, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 34.06}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.940244026482105, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.12}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68192098848522, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.565040005370975, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.64367995224893, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 34.02}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.040924886241555, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.33}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87251405790448, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 34.02}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.815281957387924, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.14}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.789925863966346, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 36.1}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.250096909701824, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.5}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15977810882032, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 35.34}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11409610696137, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39777107350528, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 35.96}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.51038703881204, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 33.85}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07501501776278, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 34.97}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43830998428166, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.56}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.839863914996386, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.269018124789, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.36}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94870595447719, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 34.99}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34580799937248, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.38}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74420311674476, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 34.82}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.88781601563096, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.98}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.78514491952956, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 35.2}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49616886489093, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.72}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.17205782979727, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38130908459425, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.64}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.270489079877734, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 35.47}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99716390110552, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.24}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62523201294243, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 34.83}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.952853901311755, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.09}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.145197089761496, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 35.33}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25159796886146, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.47}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08421781286597, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 35.36}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10994590073824, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.33}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87824511528015, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 35.09}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.120907101780176, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.44}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.609871942549944, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 34.91}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71894199214876, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.08}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85868384875357, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 35.13}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33517788350582, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.56}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75263298302889, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 35.03}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8678430095315, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.13}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75223297998309, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01674519106746, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.18}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.40389079414308, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 35.54}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25325712002814, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.57}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08417683467269, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 35.27}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.610841915011406, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.92}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.795983953401446, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 35.04}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85295209288597, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.2}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.768804002553225, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 35.01}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92693402990699, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.27}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58075088262558, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 34.91}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.830563098192215, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.1}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6527919434011, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 34.99}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.200317019596696, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.54}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.634782027453184, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 34.84}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.819771863520145, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.1}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.457602029666305, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 35.67}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.713013123720884, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.04}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.723281886428595, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 35.06}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24530688486993, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.48}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.431660970672965, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 35.75}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24473691545427, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.55}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39321993291378, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.71}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47581015340984, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.71}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.794193951413035, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 35.11}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.985794082283974, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.12}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.746663903817534, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 35.04}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29181899689138, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.606332149356604, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 35.75}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.02440711297095, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.19}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.30253006890416, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 35.64}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.812554044649005, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.06}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90177498012781, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 35.13}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55944105796516, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.81}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92120493017137, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.22}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.051914958283305, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.18}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90106507949531, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 35.07}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.671302910894156, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.83}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.56739215552807, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 35.86}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.169195944443345, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.26}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.06941700167954, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 35.4}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.763722939416766, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.9}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82799420878291, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 35.15}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39879000931978, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.61}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84452495165169, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 35.16}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.038763985037804, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.21}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86070505157113, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 35.08}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02396503649652, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.3}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.89296490140259, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 34.91}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.862812936306, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.25}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48698113486171, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 34.79}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84762306511402, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.06}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87340386770666, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 35.1}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17894712463021, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.37}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38748997077346, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 35.51}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01739478856325, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.2}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87247487343848, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.15}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4136189930141, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.68}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28887897543609, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 35.48}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.328268168494105, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.58}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51822095364332, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 34.72}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33227914199233, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.7}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.09310798533261, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.490440998226404, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.83}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.476740941405296, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 35.6}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54242090508342, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.88}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.25938911177218, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.43}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.384078139439225, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.75}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08148787356913, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 35.3}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43794001638889, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.75}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.14284689724445, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 35.34}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2001170180738, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.38}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82844380475581, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 35.12}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44047996588051, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.68}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.880704971030354, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 35.15}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42131986655295, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.65}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66267187893391, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 34.9}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.303846795111895, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.53}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58319187909365, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 34.9}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17341599985957, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.42}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67805197276175, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 34.84}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09059581160545, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.44}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.334340089932084, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.51}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27792715653777, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.64}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.741373060271144, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.08}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.809172946959734, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.17}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51227003522217, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 34.81}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6010718755424, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.84}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.857603980228305, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11088583804667, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.26}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55864081904292, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 34.74}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82674188166857, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.11}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.20362803526223, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 35.16}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14120597578585, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.36}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.03006699308753, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 35.21}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.147266091778874, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.36}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70736185833812, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 34.84}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.844542948529124, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.1}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52800007350743, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 34.8}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89554310403764, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 33.98}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98824595287442, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8604629766196, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.2}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.600761046633124, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 34.82}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02235501259565, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.39}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08796708472073, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 35.33}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.454078905284405, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.67}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.88284491747618, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 35.2}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26936713792384, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.5}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68172301352024, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 34.99}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62797196395695, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.84}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85228496417403, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 35.17}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30131802149117, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.63}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.706122966483235, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 34.91}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01889584772289, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.27}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.624371936544776, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 34.78}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59638902172446, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 33.98}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46854001656175, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 34.78}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8067619856447, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.04}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.507781060412526, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 34.76}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74080199562013, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.99}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.29292906448245, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 35.35}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83058195747435, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.08}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82448382116854, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 35.12}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83901205658913, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 33.98}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.608802085742354, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 34.87}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.5773779861629, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 39.39}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.86872090213001, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 38.67}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.60145295038819, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.77}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.866916205734015, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 36.15}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.91012498550117, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.96016697399318, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 36.13}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.61219296231866, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.78}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.84935495443642, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 36.17}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.61464303918183, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.7}], "seconds": 21.0, "failures": 0}, "rotations": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.9311578758061, "probs": [0.021278230729761302, 0.041549605758459535, 0.9371721635117792], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.60075505822897, "probs": [0.9569751557868662, 0.022187022115204392, 0.020837822097929372], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.30715894512832, "probs": [0.028764030986406904, 0.9453276881980787, 0.025908280815514436], "top": "supported", "agreement": 1.0, "input_tokens": 196}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.9599460568279, "probs": [0.9253638152569406, 0.02190751448970375, 0.052728670253355744], "top": "insufficient", "agreement": 1.0, "input_tokens": 196}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.26001802645624, "probs": [0.9597742641946788, 0.021791404944545737, 0.01843433086077556], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75721389614046, "probs": [0.0761485956363141, 0.025667636008847486, 0.8981837683548385], "top": "contradicted", "agreement": 1.0, "input_tokens": 206}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3618190344423, "probs": [0.9159676813214028, 0.0555726559991739, 0.028459662679423257], "top": "contradicted", "agreement": 1.0, "input_tokens": 208}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.06323811225593, "probs": [0.02148731809580675, 0.05171731055352475, 0.9267953713506685], "top": "insufficient", "agreement": 1.0, "input_tokens": 200}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.52810085937381, "probs": [0.025695874936561842, 0.016912927133118746, 0.9573911979303193], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.09310795553029, "probs": [0.9650011484904946, 0.014421131045601873, 0.02057772046390364], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6418321095407, "probs": [0.9685134503822489, 0.011261347946432, 0.020225201671319158], "top": "supported", "agreement": 1.0, "input_tokens": 193}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43783009052277, "probs": [0.015601263068851802, 0.9601947660643307, 0.024203970866817515], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1326459273696, "probs": [0.008885594588144235, 0.029266695521859745, 0.9618477098899961], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.55680189281702, "probs": [0.0467350628868348, 0.02343846611289458, 0.9298264710002706], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.91103602200747, "probs": [0.02661110718913202, 0.02251153411335893, 0.950877358697509], "top": "contradicted", "agreement": 1.0, "input_tokens": 199}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.98489595577121, "probs": [0.9629993612353499, 0.018113485593456775, 0.01888715317119321], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.57009186595678, "probs": [0.8820069055961203, 0.07635736188763995, 0.04163573251623969], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2659090384841, "probs": [0.02701681560472035, 0.9453954270569487, 0.027587757338330996], "top": "supported", "agreement": 1.0, "input_tokens": 198}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73464282788336, "probs": [0.02293683426823875, 0.9487912139983355, 0.028271951733425657], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.10479698516428, "probs": [0.019367602097265263, 0.013573064959569787, 0.9670593329431649], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3452689666301, "probs": [0.03987925527749218, 0.02220468207692111, 0.9379160626455868], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21147702820599, "probs": [0.02674495094579214, 0.9358820879397224, 0.03737296111448533], "top": "contradicted", "agreement": 1.0, "input_tokens": 211}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.79123417846859, "probs": [0.03051840202788244, 0.9419931233693989, 0.027488474602718756], "top": "contradicted", "agreement": 1.0, "input_tokens": 201}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.63994199037552, "probs": [0.9573434428870992, 0.022664616232544885, 0.019991940880355972], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.59219098649919, "probs": [0.20778523836492876, 0.6164398690397396, 0.17577489259533158], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.52025004290044, "probs": [0.01494019219967109, 0.958782768824845, 0.026277038975483776], "top": "permitted", "agreement": 1.0, "input_tokens": 218}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65194301307201, "probs": [0.038717451842913995, 0.06953592628537188, 0.8917466218717142], "top": "permitted", "agreement": 1.0, "input_tokens": 203}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14303599484265, "probs": [0.8769652162510786, 0.06982844563770668, 0.053206338111214745], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6419019587338, "probs": [0.8731504724710493, 0.09124479732837286, 0.03560473020057783], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.40628992021084, "probs": [0.03149783270455631, 0.9131032971636024, 0.055398870131841214], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5500500369817, "probs": [0.1973515606103041, 0.4555434753670417, 0.3471049640226543], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 209}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22493813186884, "probs": [0.02274552846853418, 0.9023369803119639, 0.07491749121950192], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.57657999359071, "probs": [0.09302877372922651, 0.07088401170776323, 0.8360872145630103], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16155697405338, "probs": [0.8635450750116728, 0.09024102515984218, 0.04621389982848493], "top": "permitted", "agreement": 1.0, "input_tokens": 211}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.57179106399417, "probs": [0.028891977481104857, 0.0412263798327744, 0.9298816426861207], "top": "permitted", "agreement": 1.0, "input_tokens": 202}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.89590505696833, "probs": [0.8174524855587436, 0.08365642576901129, 0.098891088672245], "top": "insufficient", "agreement": 1.0, "input_tokens": 204}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60132097452879, "probs": [0.017660819839882137, 0.04340570512192549, 0.9389334750381924], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.50173092633486, "probs": [0.06132539226815876, 0.8373735739130129, 0.10130103381882835], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75028299354017, "probs": [0.887606329608409, 0.06778068214997746, 0.04461298824161357], "top": "prohibited", "agreement": 1.0, "input_tokens": 213}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.2843879442662, "probs": [0.6913075067568434, 0.07074700515018907, 0.23794548809296753], "top": "insufficient", "agreement": 1.0, "input_tokens": 212}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08245711773634, "probs": [0.9055821252398959, 0.059736224410951576, 0.03468165034915258], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.52675114013255, "probs": [0.034796330955504114, 0.9473839012637663, 0.017819767780729617], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.46510014683008, "probs": [0.06676731835885638, 0.04041944853804973, 0.8928132331030938], "top": "permitted", "agreement": 1.0, "input_tokens": 208}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31987901777029, "probs": [0.0792938381036179, 0.09571489465194817, 0.8249912672444338], "top": "insufficient", "agreement": 1.0, "input_tokens": 215}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.87846487760544, "probs": [0.08689479609139293, 0.6606481239315956, 0.2524570799770115], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.10807600803673, "probs": [0.8171805101277473, 0.07074519344147624, 0.11207429643077639], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09639512747526, "probs": [0.103178626971303, 0.7367489124270009, 0.16007246060169616], "top": "prohibited", "agreement": 1.0, "input_tokens": 209}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16000702045858, "probs": [0.08560085550345033, 0.3403340342335013, 0.5740651102630484], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.49771087244153, "probs": [0.017814439341391167, 0.9671155236327101, 0.015070037025898734], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.86993512697518, "probs": [0.9513649777837366, 0.01716163808274369, 0.03147338413351958], "top": "B", "agreement": 1.0, "input_tokens": 201}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22121812775731, "probs": [0.025903351803450637, 0.9255875837884197, 0.048509064408129646], "top": "B", "agreement": 1.0, "input_tokens": 192}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4392799269408, "probs": [0.019189428631759722, 0.9189139403005832, 0.061896631067657096], "top": "A", "agreement": 1.0, "input_tokens": 192}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1016659475863, "probs": [0.016185500037668298, 0.9553471934068705, 0.028467306555461265], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14363599941134, "probs": [0.014798671264711157, 0.9697705732073082, 0.015430755527980513], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60549213550985, "probs": [0.025560818133635414, 0.022080003391839246, 0.9523591784745253], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.77850393019617, "probs": [0.05311118840492001, 0.6811115463281155, 0.2657772652669644], "top": "A", "agreement": 1.0, "input_tokens": 190}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.97032611258328, "probs": [0.026144179410345973, 0.9539350561208116, 0.019920764468842493], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.48441996797919, "probs": [0.9592258084083156, 0.024179546083237823, 0.016594645508446512], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.58193200267851, "probs": [0.015353470774838725, 0.9649135180343748, 0.019733011190786628], "top": "B", "agreement": 1.0, "input_tokens": 197}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.48660019412637, "probs": [0.6229250573303873, 0.29341037282098503, 0.08366456984862763], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54856201633811, "probs": [0.9523442934756741, 0.028977529107373938, 0.018678177416952112], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.9815650805831, "probs": [0.9509486773226812, 0.026062332698517814, 0.022988989978801007], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.50569114275277, "probs": [0.028605829867980576, 0.9599959678670565, 0.01139820226496276], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.47999991104007, "probs": [0.027524206942156276, 0.10494883289705875, 0.8675269601607849], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16861802898347, "probs": [0.025763001405599038, 0.9598922160922182, 0.014344782502182778], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08806584402919, "probs": [0.01775221293583573, 0.018510449440082843, 0.9637373376240814], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.50558101385832, "probs": [0.05805098351023624, 0.04331699912016453, 0.8986320173695993], "top": "B", "agreement": 1.0, "input_tokens": 200}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11461685039103, "probs": [0.40713917224854596, 0.04723599385140831, 0.5456248339000457], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42261879891157, "probs": [0.9628824551152317, 0.020107611199832023, 0.017009933684936158], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.20853684283793, "probs": [0.017791047056264133, 0.965845589126288, 0.016363363817447876], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.24307906255126, "probs": [0.9697536223339248, 0.01200584747578968, 0.018240530190285528], "top": "A", "agreement": 1.0, "input_tokens": 194}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32363899983466, "probs": [0.46166613268095874, 0.4246186674610029, 0.11371519985803841], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 192}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27436807751656, "probs": [0.9434733126902809, 0.03465269924248234, 0.021873988067236654], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.34514905884862, "probs": [0.05653147014528852, 0.049865123655206005, 0.8936034061995054], "top": "supported", "agreement": 1.0, "input_tokens": 214}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.10827507823706, "probs": [0.9646005420197569, 0.019726610085808626, 0.01567284789443457], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.36047886125743, "probs": [0.011200303179937986, 0.9433281975088483, 0.045471499311213706], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.99964600987732, "probs": [0.04435070832583449, 0.8823880770841831, 0.07326121458998254], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22995796054602, "probs": [0.38436219601806726, 0.39248484865749256, 0.22315295532444018], "top": "prohibited", "agreement": 0.3333333333333333, "input_tokens": 229}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4881198797375, "probs": [0.038671196720481874, 0.051824945670009545, 0.9095038576095087], "top": "permitted", "agreement": 1.0, "input_tokens": 222}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11700685694814, "probs": [0.9386225473366789, 0.024670771221418437, 0.03670668144190261], "top": "insufficient", "agreement": 1.0, "input_tokens": 221}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.22225019708276, "probs": [0.06338502047709155, 0.20445210704639485, 0.7321628724765136], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.54268383607268, "probs": [0.01023906219590939, 0.9776572412337166, 0.012103696570374124], "top": "A", "agreement": 1.0, "input_tokens": 210}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.37990997545421, "probs": [0.08343895995982653, 0.048442981731284813, 0.8681180583088886], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18730594776571, "probs": [0.028590681320234453, 0.07026846391461965, 0.9011408547651459], "top": "insufficient", "agreement": 1.0, "input_tokens": 211}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.01199604570866, "probs": [0.9485370017696566, 0.021991387483579136, 0.02947161074676416], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75802298262715, "probs": [0.1476984273576982, 0.06533845399896182, 0.78696311864334], "top": "supported", "agreement": 1.0, "input_tokens": 227}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1202560774982, "probs": [0.9659641119707486, 0.01300233393992522, 0.021033554089326202], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.45757103338838, "probs": [0.9452042665175415, 0.037745228086007734, 0.01705050539645064], "top": "insufficient", "agreement": 1.0, "input_tokens": 202}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.86861497722566, "probs": [0.7928775526060625, 0.1207272484983066, 0.08639519889563085], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3198289591819, "probs": [0.4902958520500278, 0.23093932363732236, 0.2787648243126498], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 231}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54195195436478, "probs": [0.08132590401657058, 0.05465960600149562, 0.8640144899819338], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.78537406399846, "probs": [0.2290171921944468, 0.6937872957432885, 0.07719551206226469], "top": "insufficient", "agreement": 1.0, "input_tokens": 223}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54199083708227, "probs": [0.030141465776599607, 0.9500195362761745, 0.019838997947225906], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76918302103877, "probs": [0.025305433525381633, 0.03185065213808263, 0.9428439143365358], "top": "B", "agreement": 1.0, "input_tokens": 222}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.45053007081151, "probs": [0.05059101952456338, 0.04279722164027021, 0.9066117588351665], "top": "B", "agreement": 1.0, "input_tokens": 215}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60131096281111, "probs": [0.012768350746295443, 0.01860437029559649, 0.9686272789581081], "top": "insufficient", "agreement": 1.0, "input_tokens": 201}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39680090732872, "probs": [0.9208322432392101, 0.04082517423414474, 0.038342582526645214], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.10762890428305, "probs": [0.8434852492825246, 0.05011598360172626, 0.10639876711574907], "top": "supported", "agreement": 1.0, "input_tokens": 219}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.95771600492299, "probs": [0.023619578875477214, 0.01956735197412583, 0.9568130691503969], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.32258094847202, "probs": [0.8650697558641954, 0.10248576421341996, 0.03244447992238454], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39330006577075, "probs": [0.10549476831325036, 0.0754944842886127, 0.819010747398137], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.0058269649744, "probs": [0.335350091795104, 0.11069809870287566, 0.5539518095020204], "top": "prohibited", "agreement": 1.0, "input_tokens": 221}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6362921372056, "probs": [0.8825022468959398, 0.07026933060840494, 0.047228422495655314], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2861189711839, "probs": [0.06932232571305925, 0.9077945737563922, 0.022883100530548473], "top": "insufficient", "agreement": 1.0, "input_tokens": 220}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.74048291891813, "probs": [0.009281733826954271, 0.9839356748468795, 0.006782591326166279], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.94242601655424, "probs": [0.008340312669325892, 0.9815921692199244, 0.010067518110749797], "top": "B", "agreement": 1.0, "input_tokens": 230}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.68723292089999, "probs": [0.012023447059675983, 0.9711751830662149, 0.016801369874109195], "top": "B", "agreement": 1.0, "input_tokens": 223}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3760609831661, "probs": [0.011643930956106696, 0.9806920781093156, 0.007663990934577689], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.75521786510944, "probs": [0.900478485395825, 0.023178368052708644, 0.07634314655146633], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.04082786105573, "probs": [0.06499692312355161, 0.047496249800495424, 0.887506827075953], "top": "supported", "agreement": 1.0, "input_tokens": 224}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.46219202317297, "probs": [0.3922836361905562, 0.48352898160684454, 0.12418738220259931], "top": "insufficient", "agreement": 0.3333333333333333, "input_tokens": 220}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27677896991372, "probs": [0.0560499070635644, 0.020116182060737196, 0.9238339108756985], "top": "insufficient", "agreement": 1.0, "input_tokens": 208}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.8044260609895, "probs": [0.17500642472273137, 0.21124871005668264, 0.613744865220586], "top": "permitted", "agreement": 1.0, "input_tokens": 227}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.2576320040971, "probs": [0.5914884424684399, 0.035143500631624254, 0.37336805689993585], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 228}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.43477085977793, "probs": [0.9046569284481161, 0.042704941598692904, 0.05263812995319102], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02994682267308, "probs": [0.029736313438628644, 0.033014009977779125, 0.9372496765835922], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.84194399416447, "probs": [0.7432028429041843, 0.03582480375679601, 0.2209723533390198], "top": "A", "agreement": 1.0, "input_tokens": 221}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2586298212409, "probs": [0.06632110060786917, 0.8329169964218126, 0.10076190297031828], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.04111710563302, "probs": [0.7397871533568318, 0.1860713523015022, 0.07414149434166593], "top": "A", "agreement": 1.0, "input_tokens": 207}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19589895941317, "probs": [0.9248773731002589, 0.014113602375072627, 0.06100902452466855], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93429603613913, "probs": [0.9556486965100951, 0.021248690458660385, 0.02310261303124464], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.58658295869827, "probs": [0.010853780626859795, 0.015814708468139885, 0.9733315109050004], "top": "supported", "agreement": 1.0, "input_tokens": 212}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.43129120580852, "probs": [0.9606317563420671, 0.018450800471885003, 0.020917443186047876], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22676811181009, "probs": [0.8027819128710822, 0.011505476315571077, 0.18571261081334672], "top": "contradicted", "agreement": 0.6666666666666666, "input_tokens": 203}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.46046209149063, "probs": [0.8487397319059693, 0.05482797840153974, 0.09643228969249089], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.0995280276984, "probs": [0.019700876806489794, 0.943405397557902, 0.036893725635608235], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.398029088974, "probs": [0.9481975719388965, 0.021083015776241384, 0.030719412284862045], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.84472492337227, "probs": [0.11594495476456004, 0.2790651305948641, 0.6049899146405758], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31805804930627, "probs": [0.011620895003629282, 0.009627189199890869, 0.9787519157964798], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.63498293049634, "probs": [0.9712341256474011, 0.015134214927072205, 0.013631659425526738], "top": "B", "agreement": 1.0, "input_tokens": 212}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.21632891707122, "probs": [0.9748621568921099, 0.014266993806726613, 0.010870849301163494], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11511790193617, "probs": [0.01218702517662872, 0.9640155969735945, 0.02379737784977667], "top": "insufficient", "agreement": 1.0, "input_tokens": 191}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.74359283782542, "probs": [0.05216159441960384, 0.25033305566222464, 0.6975053499181716], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65662197768688, "probs": [0.0744428867114499, 0.8598927356841719, 0.06566437760437807], "top": "supported", "agreement": 1.0, "input_tokens": 221}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42726889252663, "probs": [0.031673836031138235, 0.010902017412968387, 0.9574241465558934], "top": "supported", "agreement": 1.0, "input_tokens": 229}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.66992312669754, "probs": [0.9598638711537073, 0.029206330829344665, 0.01092979801694802], "top": "insufficient", "agreement": 1.0, "input_tokens": 226}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.48546314425766, "probs": [0.3126088342455135, 0.527299092068065, 0.1600920736864215], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.40054102987051, "probs": [0.018596715235427216, 0.8540525288330012, 0.1273507559315715], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5070508737117, "probs": [0.2682095319185024, 0.5694223884119577, 0.16236807966953995], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32582900486887, "probs": [0.5197715159798874, 0.040530340274099394, 0.4396981437460133], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 224}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.28891906142235, "probs": [0.03677949530401433, 0.9404844838908847, 0.02273602080510092], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19812698476017, "probs": [0.09722552658518739, 0.7089119171844285, 0.19386255623038412], "top": "B", "agreement": 1.0, "input_tokens": 218}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27174803428352, "probs": [0.0446355264117619, 0.029378934066784906, 0.9259855395214532], "top": "B", "agreement": 1.0, "input_tokens": 209}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30207795090973, "probs": [0.8956127457317613, 0.023540301062398936, 0.08084695320583983], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.97307607531548, "probs": [0.9215420659013949, 0.05709241087032677, 0.021365523228278453], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.80536397174001, "probs": [0.024650674719169527, 0.05690076079826144, 0.9184485644825692], "top": "contradicted", "agreement": 1.0, "input_tokens": 215}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.16286913864315, "probs": [0.017881542110937673, 0.0314503320010902, 0.9506681258879721], "top": "contradicted", "agreement": 1.0, "input_tokens": 236}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27194803580642, "probs": [0.015971281239954956, 0.021403804942628207, 0.9626249138174169], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19308915920556, "probs": [0.9469841068616602, 0.027062216114961006, 0.025953677023378714], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42057012207806, "probs": [0.9655262695437641, 0.017417092730978554, 0.01705663772525738], "top": "supported", "agreement": 1.0, "input_tokens": 241}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76190403662622, "probs": [0.9523229876734454, 0.019887100792948465, 0.02778991153360623], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.61966220848262, "probs": [0.02944395390697739, 0.022909170093210977, 0.9476468759998116], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.71019304916263, "probs": [0.028467305657075486, 0.01618550022217239, 0.9553471941207522], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00065493397415, "probs": [0.9626180927581529, 0.027508993151357353, 0.009872914090489732], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22309713996947, "probs": [0.01216177373585067, 0.025820060282101747, 0.9620181659820476], "top": "supported", "agreement": 1.0, "input_tokens": 216}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.36040901206434, "probs": [0.008947931903485177, 0.02245643770922697, 0.9685956303872878], "top": "supported", "agreement": 1.0, "input_tokens": 237}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75507394969463, "probs": [0.03490400868942244, 0.07256938070358598, 0.8925266106069916], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65200285054743, "probs": [0.8560321676779682, 0.10355825471147521, 0.04040957761055657], "top": "contradicted", "agreement": 1.0, "input_tokens": 209}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.84476497024298, "probs": [0.9275028632254695, 0.04760344873379892, 0.024893688040731596], "top": "contradicted", "agreement": 1.0, "input_tokens": 230}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.21488885954022, "probs": [0.9261949727483907, 0.021028996331540862, 0.0527760309200684], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.33013800159097, "probs": [0.03674381458625419, 0.02368412668428037, 0.9395720587294654], "top": "supported", "agreement": 1.0, "input_tokens": 222}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43644009158015, "probs": [0.026670715956627202, 0.020321964093249996, 0.9530073199501229], "top": "supported", "agreement": 1.0, "input_tokens": 243}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.55380093865097, "probs": [0.08330838841015804, 0.7031959025468014, 0.21349570904304058], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.69396312348545, "probs": [0.18172827933132565, 0.6645395854888777, 0.15373213517979664], "top": "prohibited", "agreement": 1.0, "input_tokens": 223}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.42154118977487, "probs": [0.08895150060315864, 0.816337598890086, 0.09471090050675535], "top": "prohibited", "agreement": 1.0, "input_tokens": 244}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19060904718935, "probs": [0.0293692258945804, 0.11923451516955454, 0.851396258935865], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.77500308863819, "probs": [0.7559638022373625, 0.22951644325448917, 0.014519754508148302], "top": "permitted", "agreement": 1.0, "input_tokens": 220}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.29571818187833, "probs": [0.9180643670041458, 0.055699863652112935, 0.026235769343741142], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.63435219228268, "probs": [0.8545877320672847, 0.07877388866940015, 0.06663837926331509], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.06825817376375, "probs": [0.10813821274832802, 0.06970313933726799, 0.822158647914404], "top": "prohibited", "agreement": 1.0, "input_tokens": 219}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.06301918253303, "probs": [0.04460783183966404, 0.029981179056752127, 0.9254109891035839], "top": "prohibited", "agreement": 1.0, "input_tokens": 240}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.11868805810809, "probs": [0.9383472668786358, 0.03989758917731366, 0.021755143944050585], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.81559408456087, "probs": [0.019454336721104887, 0.06822606373845527, 0.9123195995404397], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.77622312121093, "probs": [0.015551699476841353, 0.0471124973010445, 0.9373358032221141], "top": "permitted", "agreement": 1.0, "input_tokens": 246}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.85116501897573, "probs": [0.0377743579001357, 0.05503984914430352, 0.9071857929555608], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18978792242706, "probs": [0.9057645436591656, 0.057300812015498664, 0.03693464432533565], "top": "prohibited", "agreement": 1.0, "input_tokens": 227}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73436296544969, "probs": [0.9388430475426657, 0.037491202062820105, 0.023665750394514205], "top": "prohibited", "agreement": 1.0, "input_tokens": 248}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.61249195598066, "probs": [0.28824200219090185, 0.5868851532425197, 0.12487284456657842], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02542711421847, "probs": [0.5631497490596038, 0.19793031332732086, 0.2389199376130753], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.01386909931898, "probs": [0.07442810557243966, 0.7426469601625133, 0.18292493426504697], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.25026910565794, "probs": [0.021424613092898774, 0.963560742046841, 0.01501464486026032], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.77597399242222, "probs": [0.01601169462902707, 0.9650607227828113, 0.01892758258816167], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.23555800318718, "probs": [0.01772358905716697, 0.9621833982502599, 0.020093012692573263], "top": "A", "agreement": 1.0, "input_tokens": 232}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.58075201697648, "probs": [0.038484166247206124, 0.9437641516667067, 0.017751682086087198], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93787604011595, "probs": [0.01638885815779191, 0.9473306235405403, 0.036280518301667765], "top": "B", "agreement": 1.0, "input_tokens": 219}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75703391805291, "probs": [0.01830968230910091, 0.9532844759985375, 0.028405841692361555], "top": "B", "agreement": 1.0, "input_tokens": 240}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.59686203487217, "probs": [0.023557953254122476, 0.9543166602232142, 0.022125386522663444], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08357593789697, "probs": [0.031413960782038314, 0.9495687268473656, 0.01901731237059622], "top": "A", "agreement": 1.0, "input_tokens": 214}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.2764281630516, "probs": [0.07602225290100428, 0.8966935357050799, 0.027284211393915826], "top": "A", "agreement": 1.0, "input_tokens": 235}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.88685609400272, "probs": [0.023803230827026532, 0.0318997393139744, 0.944297029858999], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.55451200343668, "probs": [0.9537738809196197, 0.029021027054749942, 0.017205092025630454], "top": "B", "agreement": 1.0, "input_tokens": 214}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.45048001222312, "probs": [0.964600540916227, 0.019726610712202018, 0.015672848371570857], "top": "B", "agreement": 1.0, "input_tokens": 235}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.4741020090878, "probs": [0.01331158113941839, 0.9684719956116256, 0.018216423248955952], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.48543201386929, "probs": [0.03555637420973125, 0.9480424533538477, 0.016401172436420974], "top": "A", "agreement": 1.0, "input_tokens": 219}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.96605692990124, "probs": [0.02192392382604651, 0.9656109109209706, 0.012465165252982892], "top": "A", "agreement": 1.0, "input_tokens": 240}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.71126313880086, "probs": [0.019595497666027976, 0.022215172638678914, 0.9581893296952932], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54134217090905, "probs": [0.9596984415322811, 0.02225016024721796, 0.01805139822050088], "top": "B", "agreement": 1.0, "input_tokens": 217}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.96467601135373, "probs": [0.9679216672147071, 0.013872260676720507, 0.01820607210857243], "top": "B", "agreement": 1.0, "input_tokens": 238}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60257197357714, "probs": [0.02350526733620643, 0.04401818593722302, 0.9324765467265705], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3423699233681, "probs": [0.9307059696632914, 0.042134929049560656, 0.027159101287147974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.10237694345415, "probs": [0.9518944397002193, 0.030200951503065683, 0.01790460879671502], "top": "contradicted", "agreement": 1.0, "input_tokens": 248}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43862986378372, "probs": [0.0734946189438627, 0.8668797160257922, 0.05962566503034527], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.81734403967857, "probs": [0.06641491931728377, 0.8517220237244112, 0.08186305695830509], "top": "prohibited", "agreement": 1.0, "input_tokens": 239}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.02240793034434, "probs": [0.026075603104276424, 0.9317425408247122, 0.04218185607101137], "top": "prohibited", "agreement": 1.0, "input_tokens": 260}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43583892285824, "probs": [0.8272206536224163, 0.11345127657827278, 0.05932806979931101], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.16133804433048, "probs": [0.10398356939461242, 0.36466878809364145, 0.5313476425117462], "top": "A", "agreement": 0.3333333333333333, "input_tokens": 227}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8180439285934, "probs": [0.047442131153065406, 0.31816121638654793, 0.6343966524603867], "top": "A", "agreement": 1.0, "input_tokens": 248}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.44313087686896, "probs": [0.026110501601006305, 0.021183264898242012, 0.9527062335007518], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8408641256392, "probs": [0.9405169120595193, 0.024208959715331872, 0.035274128225148874], "top": "contradicted", "agreement": 1.0, "input_tokens": 237}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.11466893553734, "probs": [0.9538802773453129, 0.023059861528293313, 0.023059861126393875], "top": "contradicted", "agreement": 1.0, "input_tokens": 258}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11869813874364, "probs": [0.11358428062013931, 0.09215012355248668, 0.7942655958273741], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.67206190899014, "probs": [0.6104725669434667, 0.2700592902850353, 0.11946814277149793], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 244}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.08886897191405, "probs": [0.8448473542913075, 0.08645996139090424, 0.06869268431778815], "top": "prohibited", "agreement": 1.0, "input_tokens": 265}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.84228485822678, "probs": [0.0265024583113502, 0.9469950831682415, 0.02650245852040832], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19627917185426, "probs": [0.04213492809402675, 0.9307059697792428, 0.027159102126730544], "top": "A", "agreement": 1.0, "input_tokens": 233}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.46868194453418, "probs": [0.02140380544189133, 0.962624913264518, 0.01597128129359056], "top": "A", "agreement": 1.0, "input_tokens": 254}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.30653006955981, "probs": [0.04515777084298974, 0.03741040508103143, 0.9174318240759788], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.05001682415605, "probs": [0.9100932546372905, 0.041201749446211355, 0.048704995916498045], "top": "contradicted", "agreement": 1.0, "input_tokens": 234}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.30667908117175, "probs": [0.9366656772497187, 0.038194709842211357, 0.025139612908070024], "top": "contradicted", "agreement": 1.0, "input_tokens": 255}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.09958809800446, "probs": [0.868308675840529, 0.07676005878826689, 0.0549312653712041], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75233399868011, "probs": [0.15789450752928103, 0.11538080349900755, 0.7267246889717114], "top": "prohibited", "agreement": 1.0, "input_tokens": 235}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70547310635448, "probs": [0.08083885614456791, 0.06032105358305889, 0.8588400902723732], "top": "prohibited", "agreement": 1.0, "input_tokens": 256}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.00184602662921, "probs": [0.009236476474052145, 0.9791380440055677, 0.011625479520380164], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.7162331417203, "probs": [0.014594034131288612, 0.9765710625943955, 0.008834903274315823], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.43614309467375, "probs": [0.0070908823458139675, 0.986522232315347, 0.006386885338839195], "top": "A", "agreement": 1.0, "input_tokens": 259}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.14872903190553, "probs": [0.07423043782078662, 0.012813700792315847, 0.9129558613868974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.30960103683174, "probs": [0.9080827305344887, 0.025413463174314043, 0.06650380629119718], "top": "contradicted", "agreement": 1.0, "input_tokens": 238}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.67158586531878, "probs": [0.912724042631006, 0.024497031790175153, 0.06277892557881881], "top": "contradicted", "agreement": 1.0, "input_tokens": 259}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.0518070589751, "probs": [0.44029144656423663, 0.4049592579435585, 0.15474929549220479], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 227}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.91394593939185, "probs": [0.36520001887064335, 0.21202778871080402, 0.4227721924185527], "top": "permitted", "agreement": 0.3333333333333333, "input_tokens": 238}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.95651709102094, "probs": [0.0870101345530018, 0.5040542344698868, 0.4089356309771113], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 259}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.72481497749686, "probs": [0.4770731173522989, 0.04585376828987368, 0.4770731143578275], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 221}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.46735201589763, "probs": [0.39895641057043185, 0.0317669442524896, 0.5692766451770785], "top": "B", "agreement": 1.0, "input_tokens": 232}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.9975759126246, "probs": [0.3779951490486449, 0.03557902847506642, 0.5864258224762886], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 253}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76963487640023, "probs": [0.02269220266989126, 0.01879907857817761, 0.9585087187519311], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.66751395910978, "probs": [0.9501164329419498, 0.021572126471092922, 0.028311440586957334], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02874704636633, "probs": [0.9561058300755273, 0.022635315404623546, 0.021258854519849178], "top": "contradicted", "agreement": 1.0, "input_tokens": 245}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.79177497513592, "probs": [0.09448034891197682, 0.0384419614966505, 0.8670776895913728], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.25109914690256, "probs": [0.7989049560014004, 0.057297274549302536, 0.143797769449297], "top": "prohibited", "agreement": 1.0, "input_tokens": 229}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93339591287076, "probs": [0.9033714574378554, 0.03317980421079573, 0.06344873835134875], "top": "prohibited", "agreement": 1.0, "input_tokens": 250}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.06735711917281, "probs": [0.9807799827157122, 0.008509520309325268, 0.010710496974962463], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.87821598164737, "probs": [0.009842123709939055, 0.01026250243418525, 0.9798953738558757], "top": "A", "agreement": 1.0, "input_tokens": 217}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.25776904635131, "probs": [0.011627702527445261, 0.00904705287357599, 0.9793252445989787], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.06572707183659, "probs": [0.6618883595686608, 0.28081044656836285, 0.05730119386297648], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.51785095594823, "probs": [0.06488320495606428, 0.37587178359700457, 0.5592450114469311], "top": "contradicted", "agreement": 1.0, "input_tokens": 235}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76298593170941, "probs": [0.06100397866886853, 0.1887116244179442, 0.7502843969131873], "top": "contradicted", "agreement": 1.0, "input_tokens": 256}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.7609140407294, "probs": [0.17125015092489698, 0.5080506282282206, 0.32069922084688257], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.64576315321028, "probs": [0.1514529519738628, 0.627869626193126, 0.22067742183301117], "top": "permitted", "agreement": 1.0, "input_tokens": 237}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.44840302504599, "probs": [0.17013095481276255, 0.7202069629772057, 0.10966208221003176], "top": "permitted", "agreement": 1.0, "input_tokens": 258}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19406797923148, "probs": [0.023818955965634256, 0.9449208465220887, 0.031260197512277065], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08713701367378, "probs": [0.04244531776287583, 0.9375620734741275, 0.01999260876299673], "top": "A", "agreement": 1.0, "input_tokens": 224}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6215821299702, "probs": [0.028930382730149296, 0.9507948329241313, 0.02027478434571944], "top": "A", "agreement": 1.0, "input_tokens": 245}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.81614607200027, "probs": [0.227340819830238, 0.7726591801697621], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.70550401881337, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97137705609202, "probs": [0.16800751383797227, 0.8319924861620277], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36323016695678, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.81655608676374, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30363816395402, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.44319096207619, "probs": [0.20097170058635486, 0.7990282994136452], "top": "ordinary", "agreement": 1.0, "input_tokens": 254}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.37182115204632, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.55417201668024, "probs": [0.13950622912843208, 0.8604937708715679], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9347361009568, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.34694994986057, "probs": [0.3067002008775075, 0.6932997991224925], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.51691988110542, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.32105808705091, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "agreement": 1.0, "input_tokens": 256}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62251207046211, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.23809906281531, "probs": [0.07740461434958126, 0.9225953856504188], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80704385973513, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.12803792208433, "probs": [0.09196032674706488, 0.9080396732529351], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.49413996376097, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.4212711211294, "probs": [0.1552608265624769, 0.844739173437523], "top": "ordinary", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.06008497066796, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.2641998436302, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63545209914446, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56350091099739, "probs": [0.32019975839988934, 0.6798002416001107], "top": "ordinary", "agreement": 1.0, "input_tokens": 255}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52595091611147, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.34317995607853, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.47638213448226, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.91049493849277, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16428802348673, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.91245693340898, "probs": [0.13578298208575287, 0.8642170179142472], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.45154096744955, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.25024909712374, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.78739388473332, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.13811918720603, "probs": [0.9710643563300382, 0.02893564366996184], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.5520420819521, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.94593704864383, "probs": [0.9673224283590844, 0.032677571640915566], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.2310789860785, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.4051611032337, "probs": [0.9350120677910418, 0.06498793220895807], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.3602909129113, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.30336917564273, "probs": [0.8405803537312423, 0.1594196462687578], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.56977213546634, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.96124706044793, "probs": [0.9290542127600319, 0.07094578723996812], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.4266511388123, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.07990896515548, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.61306303553283, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.81683415547013, "probs": [0.9516645319845058, 0.048335468015494064], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.09091500751674, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.78955408744514, "probs": [0.9439168350492196, 0.056083164950780265], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.59583107568324, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72997295856476, "probs": [0.905386513027929, 0.09461348697207088], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84875407256186, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80579286068678, "probs": [0.9156019647894573, 0.0843980352105427], "top": "react", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56023103743792, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02362711541355, "probs": [0.6155100736384166, 0.38448992636158347], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4560798369348, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.10239814594388, "probs": [0.6932998086882517, 0.30670019131174836], "top": "react", "agreement": 1.0, "input_tokens": 268}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.19050691835582, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.61060294695199, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.28141797147691, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97224604897201, "probs": [0.915601962671152, 0.08439803732884785], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47931098006666, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31606007553637, "probs": [0.40698895046193323, 0.5930110495380667], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79385400377214, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1342891920358, "probs": [0.36247441705939704, 0.637525582940603], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63316197693348, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.35913001932204, "probs": [0.908039679856927, 0.09196032014307312], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52918097563088, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31062999926507, "probs": [0.3481052260909569, 0.6518947739090432], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03880581073463, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7315231449902, "probs": [0.5235093404303358, 0.4764906595696642], "top": "react", "agreement": 0.5, "input_tokens": 255}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.49948016740382, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80305384099483, "probs": [0.2743951410804348, 0.7256048589195652], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30645797587931, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62990188226104, "probs": [0.2112354509623397, 0.7887645490376604], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.92761613987386, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.01358606107533, "probs": [0.12511350715617336, 0.8748864928438266], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36463110893965, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64371199905872, "probs": [0.27439514286499256, 0.7256048571350073], "top": "ordinary", "agreement": 1.0, "input_tokens": 181}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85502490028739, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 78.41397006995976, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.4458459559828, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.7475969363004, "probs": [0.4222145135481339, 0.5777854864518661], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.43509187921882, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.25487001799047, "probs": [0.1472092305510008, 0.8527907694489993], "top": "ordinary", "agreement": 1.0, "input_tokens": 183}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.7827160153538, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.23136886022985, "probs": [0.11201237833636817, 0.8879876216636319], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.34923099167645, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.12789682671428, "probs": [0.12172004831180805, 0.878279951688192], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86775491572917, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.6236219741404, "probs": [0.17243807104776526, 0.8275619289522348], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72852288931608, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48674014024436, "probs": [0.16366828025224447, 0.8363317197477556], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.31220796145499, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64794206619263, "probs": [0.3000708042310285, 0.6999291957689716], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.1797361727804, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.57711195759475, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47368998266757, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0013570971787, "probs": [0.10592235643160026, 0.8940776435683997], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7030530795455, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.41902903094888, "probs": [0.19108583142179145, 0.8089141685782085], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7184329405427, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.49457202851772, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61959097720683, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73956301994622, "probs": [0.7614507841997993, 0.23854921580020058], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64914207533002, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80460402742028, "probs": [0.7887645476620578, 0.2112354523379421], "top": "react", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.36233896017075, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8806351069361, "probs": [0.4921584130878123, 0.5078415869121877], "top": "ordinary", "agreement": 0.5, "input_tokens": 191}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61522191017866, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7573028523475, "probs": [0.293524032474578, 0.7064759675254221], "top": "ordinary", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.2042580600828, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.6900029219687, "probs": [0.5156793119007109, 0.484320688099289], "top": "react", "agreement": 0.5, "input_tokens": 186}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.19140704162419, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64793205447495, "probs": [0.6302458313412554, 0.3697541686587445], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56858080998063, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.33609894663095, "probs": [0.7318060925310856, 0.2681939074689144], "top": "react", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.36776903644204, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.32456002384424, "probs": [0.5469149637193962, 0.45308503628060387], "top": "react", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73882285133004, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.45701491087675, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1260670106858, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.87936478294432, "probs": [0.44532390802961846, 0.5546760919703815], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0897070132196, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76501303911209, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97557599283755, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16579909436405, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "agreement": 1.0, "input_tokens": 195}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79139414802194, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.32578804157674, "probs": [0.2501313189948441, 0.749868681005156], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52015110477805, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.81100407615304, "probs": [0.17696061199867663, 0.8230393880013234], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43688900396228, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27766706980765, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06822698935866, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62396190688014, "probs": [0.2870615213045163, 0.7129384786954838], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62364199757576, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.65142195299268, "probs": [0.4145813330827104, 0.5854186669172896], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.83310412801802, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73094293102622, "probs": [0.44532391331269805, 0.554676086687302], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.53418101370335, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85972388833761, "probs": [0.35525721497515955, 0.6447427850248404], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.26295799389482, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.76245595514774, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77425385452807, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.00504893995821, "probs": [0.8269315768853935, 0.12075476980815658, 0.05231365330644987], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25183798745275, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97263806685805, "probs": [0.9608203282304785, 0.03917967176952151], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84254401363432, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.68104710243642, "probs": [0.9620325019409397, 0.02528607877990563, 0.012681419279154706], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.1371790021658, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87276606075466, "probs": [0.06890580795431694, 0.931094192045683], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1965089906007, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.47688419744372, "probs": [0.8320645593484999, 0.04643111322039759, 0.12150432743110248], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.51550185866654, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.40375194326043, "probs": [0.9544700034284429, 0.04552999657155723], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61462097428739, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.44388417899609, "probs": [0.9527150999124248, 0.019483546419057494, 0.027801353668517746], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.98256781324744, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.17453995347023, "probs": [0.12172004535178665, 0.8782799546482134], "top": "holds", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8056031037122, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.02636877633631, "probs": [0.0645643144235774, 0.035205281163747985, 0.9002304044126745], "top": "tavern", "agreement": 1.0, "input_tokens": 400}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5746111087501, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.11093900352716, "probs": [0.1862839992992625, 0.8137160007007376], "top": "elsewhere", "agreement": 1.0, "input_tokens": 360}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48598995991051, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.60709589906037, "probs": [0.36775256751831553, 0.024771247006193476, 0.6074761854754909], "top": "tavern", "agreement": 0.6666666666666666, "input_tokens": 412}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.35808110237122, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.91875686496496, "probs": [0.0796746792375593, 0.9203253207624406], "top": "holds", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.42368890345097, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.637526165694, "probs": [0.08290919036094808, 0.8994500391535621, 0.017640770485489753], "top": "meet_wren", "agreement": 1.0, "input_tokens": 403}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.63768190331757, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.99066800624132, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 363}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.37686898931861, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.47616498358548, "probs": [0.6296525959845715, 0.35799812782967866, 0.012349276185749847], "top": "stay", "agreement": 0.6666666666666666, "input_tokens": 415}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.29781987518072, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.70926400087774, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 351}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67806290090084, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.16596099175513, "probs": [0.5514055185500297, 0.41145265329172886, 0.03714182815824141], "top": "stay", "agreement": 1.0, "input_tokens": 404}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.05312697589397, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.77418486773968, "probs": [0.9673224283553485, 0.03267757164465143], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.94348008185625, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.11410301923752, "probs": [0.9603331749383929, 0.028023658411333054, 0.011643166650274083], "top": "stay", "agreement": 1.0, "input_tokens": 416}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.6832048818469, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.22827097959816, "probs": [0.7439389594906934, 0.2560610405093065], "top": "moved", "agreement": 1.0, "input_tokens": 345}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8839738983661, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.38077310472727, "probs": [0.329927824572513, 0.5565122407492623, 0.11355993467822469], "top": "meet_wren", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.7971939444542, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.65034387819469, "probs": [0.02983020500731863, 0.9701697949926814], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22392789460719, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.75856806524098, "probs": [0.7871268373024943, 0.18209101999191218, 0.030782142705593513], "top": "stay", "agreement": 1.0, "input_tokens": 410}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.70540505088866, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97765812836587, "probs": [0.04979902878533396, 0.950200971214666], "top": "holds", "agreement": 1.0, "input_tokens": 334}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.40312902629375, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.00097999162972, "probs": [0.9109478037576992, 0.057628718316645906, 0.03142347792565483], "top": "stay", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.03096708655357, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.96929811127484, "probs": [0.9719328658769364, 0.028067134123063546], "top": "same", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.5482709929347, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.23186207562685, "probs": [0.9710720272034978, 0.01715460731299303, 0.01177336548350914], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.34012101218104, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.84426596015692, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.95595605298877, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.92466788552701, "probs": [0.1337836073050901, 0.8426345520770127, 0.0235818406178973], "top": "meet_wren", "agreement": 1.0, "input_tokens": 407}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27277807705104, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.05777817964554, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18641678243876, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.83876914158463, "probs": [0.7852826544019835, 0.20168841824349107, 0.013028927354525465], "top": "stay", "agreement": 1.0, "input_tokens": 419}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.35854001156986, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.70830380730331, "probs": [0.05776707671018535, 0.9422329232898147], "top": "holds", "agreement": 1.0, "input_tokens": 336}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.69693289324641, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.56299303844571, "probs": [0.706475960089318, 0.29352403991068193], "top": "stay", "agreement": 1.0, "input_tokens": 370}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.03605498559773, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.66768394038081, "probs": [0.9631149772558724, 0.036885022744127596], "top": "same", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.6138519346714, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.91637710295618, "probs": [0.9583890947004433, 0.04161090529955677], "top": "stay", "agreement": 1.0, "input_tokens": 381}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.5907709673047, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.11527989804745, "probs": [0.05608316364845092, 0.9439168363515491], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9418351072818, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98823702149093, "probs": [0.5156793134278208, 0.4843206865721793], "top": "stay", "agreement": 0.5, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47136982530355, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.90227594785392, "probs": [0.9471470205154783, 0.05285297948452184], "top": "same", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48821000009775, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.24167084880173, "probs": [0.9471470206637833, 0.05285297933621657], "top": "stay", "agreement": 1.0, "input_tokens": 389}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.44321012310684, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.69086502492428, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 340}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06865702755749, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.89850595407188, "probs": [0.7614507825686178, 0.23854921743138224], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.65045291185379, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 76.00817200727761, "probs": [0.9583890946533792, 0.04161090534662085], "top": "same", "agreement": 1.0, "input_tokens": 353}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61940215155482, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.22157081216574, "probs": [0.9516645320340041, 0.04833546796599583], "top": "stay", "agreement": 1.0, "input_tokens": 385}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.94763491861522, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.80429592356086, "probs": [0.4921584129984156, 0.5078415870015844], "top": "holds", "agreement": 1.0, "input_tokens": 333}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84023386798799, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.31325090490282, "probs": [0.07094578575062087, 0.9290542142493791], "top": "departure", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52715092524886, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76106509380043, "probs": [0.0820053924809899, 0.9179946075190101], "top": "elsewhere", "agreement": 1.0, "input_tokens": 346}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61790202371776, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.1454489286989, "probs": [0.1724380702954841, 0.8275619297045159], "top": "departure", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85507495887578, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.64024415053427, "probs": [0.26819391433993306, 0.7318060856600669], "top": "holds", "agreement": 1.0, "input_tokens": 339}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.83387409895658, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.20461003109813, "probs": [0.04833546845435023, 0.9516645315456497], "top": "departure", "agreement": 1.0, "input_tokens": 373}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52672996744514, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.95395713299513, "probs": [0.06310762735784174, 0.9368923726421583], "top": "elsewhere", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79361395537853, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.03937803953886, "probs": [0.17243806760921704, 0.8275619323907829], "top": "departure", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.2880389764905, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.67508399672806, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 337}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.87440409325063, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.95692712068558, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "agreement": 1.0, "input_tokens": 371}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67476206086576, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.08072806335986, "probs": [0.9608203281857868, 0.039179671814213245], "top": "same", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.71344291418791, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.50636187382042, "probs": [0.9439168372028508, 0.05608316279714923], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30484888330102, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.88376591168344, "probs": [0.07740461992444837, 0.9225953800755516], "top": "holds", "agreement": 1.0, "input_tokens": 343}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.96541596204042, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.92132708244026, "probs": [0.6589789156058212, 0.3410210843941787], "top": "stay", "agreement": 1.0, "input_tokens": 377}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.60508190281689, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.05269804783165, "probs": [0.9596222367905244, 0.040377763209475684], "top": "same", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62751187570393, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.36748205125332, "probs": [0.9422329206296081, 0.05776707937039187], "top": "stay", "agreement": 1.0, "input_tokens": 388}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.74603291787207, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.10590893030167, "probs": [0.061278160236647276, 0.9387218397633527], "top": "holds", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.561821969226, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.12628882937133, "probs": [0.8910699865839312, 0.10893001341606875], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48626004345715, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.00605694763362, "probs": [0.9663161960398492, 0.03368380396015082], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.28472812473774, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.09722900390625, "probs": [0.9673224283958624, 0.03267757160413767], "top": "stay", "agreement": 1.0, "input_tokens": 394}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38722902163863, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.55540206097066, "probs": [0.8527907664515249, 0.14720923354847515], "top": "moved", "agreement": 1.0, "input_tokens": 348}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.07277696579695, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.03670817054808, "probs": [0.04161090472473705, 0.9583890952752631], "top": "outside", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67454180307686, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.58613291196525, "probs": [0.05608316279714923, 0.9439168372028508], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.33416901342571, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.51858401671052, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "agreement": 1.0, "input_tokens": 390}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.58680190332234, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.28763091936707, "probs": [0.07094578723996812, 0.9290542127600319], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73850387334824, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.44053602032363, "probs": [0.8040183777913431, 0.19598162220865686], "top": "stay", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.53210090473294, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.76881595700979, "probs": [0.959622237525547, 0.040377762474453], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7839440330863, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.15545103140175, "probs": [0.9583890941641539, 0.041610905835846104], "top": "stay", "agreement": 1.0, "input_tokens": 396}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43114996142685, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.65695596672595, "probs": [0.7781221683337017, 0.22187783166629826], "top": "moved", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89579494297504, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.84745706804097, "probs": [0.21123545145303588, 0.7887645485469642], "top": "outside", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.54027093201876, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.21802014485002, "probs": [0.03688502246073314, 0.9631149775392668], "top": "elsewhere", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.54820114374161, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.32473201677203, "probs": [0.755706186939545, 0.244293813060455], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.5916610788554, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98313803039491, "probs": [0.12172004614068278, 0.8782799538593171], "top": "holds", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03498692251742, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.51139304228127, "probs": [0.8879876230204682, 0.11201237697953184], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86797400936484, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.23015108704567, "probs": [0.9558139330893122, 0.04418606691068789], "top": "same", "agreement": 1.0, "input_tokens": 366}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38847885653377, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.4776530880481, "probs": [0.9631149782667298, 0.0368850217332701], "top": "stay", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7676340136677, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.27657099813223, "probs": [0.09733498488397495, 0.9026650151160249], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.48021196573973, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.23026098310947, "probs": [0.8319924818822048, 0.16800751811779524], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30693783983588, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.61691312305629, "probs": [0.9471470206637833, 0.05285297933621657], "top": "same", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.70864397287369, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.23966191709042, "probs": [0.955813931654481, 0.044186068345518914], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.4672821816057, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}], "seconds": 50.2, "failures": 0}, "repeat": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70988201722503, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 34.86}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.146916845813394, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.49}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90999397076666, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.2}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.416309118270874, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.68}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77647188305855, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.1}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10887604393065, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.45}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.756310844793916, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.03}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35940903425217, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.72}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44161897338927, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.73}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.374718114733696, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.46}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.921653896570206, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.19}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2343179769814, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 34.43}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06369502469897, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.23}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08434593863785, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.31}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01336495578289, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 34.28}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16421592980623, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.46}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.004035199061036, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.35}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24762797355652, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.61}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0995260309428, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.39}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36093803308904, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.71}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.776121938601136, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.09}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.290448090061545, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.62}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848583025857806, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.16}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.089415825903416, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.46}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87001299113035, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.19}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.337249144911766, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.58}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88288293778896, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.31}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.984045058488846, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.33}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.816392093896866, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.04}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.229936104267836, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 34.51}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7216719314456, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 33.98}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19411697238684, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.47}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.006294121965766, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.3}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08788612857461, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.36}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86407394707203, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.16}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33314806595445, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.59}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73179214820266, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 33.99}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46828017756343, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.72}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.778071196749806, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.07}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.380068795755506, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.71}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8168018758297, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.14}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06275508739054, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.37}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44653007015586, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.69}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28349693119526, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.52}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.58098983578384, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 33.89}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18775697052479, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.47}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.979263881221414, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.31}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08615619689226, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.44}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.374141953885555, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 35.8}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59908296726644, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.96}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57010097615421, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.84}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.411479046568274, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.65}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83076286688447, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.11}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31487784534693, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.54}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.773551953956485, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.01}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.065764889121056, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.4}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04106481745839, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.33}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25965693593025, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.59}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92886303178966, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.18}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54543117247522, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.79}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83672193437815, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.14}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25805808790028, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.61}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.884263856336474, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.2}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.004204001277685, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.35}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85065312311053, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.16}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.077485118061304, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.41}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69494087807834, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.04}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.709482945501804, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 35.01}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.048056023195386, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.23}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09805593825877, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.43}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.155895959585905, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.46}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12352621369064, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.48}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87720303423703, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.17}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17802604660392, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.51}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.000255193561316, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.2}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.409848999232054, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.72}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57108096033335, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 34.8}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87606488913298, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 35.22}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73804404772818, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 35.03}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7985539380461, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 35.14}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09947597235441, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.45}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.133725825697184, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.38}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.681801080703735, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.04}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11712593212724, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.47}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73692210577428, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.06}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.126346956938505, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.38}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05012612044811, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.38}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.155646132305264, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.49}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.941254045814276, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.13}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43304987438023, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.52}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94966295734048, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.12}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79758303426206, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 34.97}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98141383938491, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.02}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.346098106354475, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.76}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.671011010184884, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.12}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06412483006716, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.39}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.778842099010944, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.3}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39385912194848, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.88}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89880413003266, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.4}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44183899834752, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.97}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90252406522632, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.41}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.370498990640044, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.87}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90224506519735, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 35.47}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.941914819180965, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.31}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12329478003085, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.63}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63611218892038, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 35.02}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11545607075095, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.59}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37785900011659, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.93}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.907003028318286, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.36}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4831510707736, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 35.03}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15803590603173, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.61}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04306483268738, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.48}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.490428959950805, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 33.94}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67321212217212, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 35.16}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84211289510131, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.35}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68425318412483, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 35.19}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.216536002233624, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.65}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91224381327629, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.34}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10261592641473, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.62}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39791898988187, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.8}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99517482891679, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.53}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32543811202049, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.87}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0525848120451, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.58}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44040010496974, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.89}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34808787703514, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.87}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.419399013742805, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.88}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12081606499851, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.67}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.273206980898976, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.79}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90733411349356, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.35}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73767384700477, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 35.14}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.50076896883547, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 33.97}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.319008028134704, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.8}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97767504490912, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.51}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41794894449413, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.93}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7022508289665, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.2}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43770997971296, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.97}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87226283363998, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.38}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.425520131364465, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.98}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87449311837554, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.36}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.402088986709714, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.91}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99278389103711, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.52}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2107969596982, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.68}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92696406506002, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.44}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17653709463775, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.69}], "seconds": 5.4, "failures": 0}, "reversed": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.101916037499905, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.57}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55410993471742, "probs": [0.02983410046106534, 0.02983410046106534, 0.9403317990778695], "top": "supported", "option_logits": null, "prompt_sha256": "44861e558c186b2b16dcf17748026cc812485bd237c1ed73b0e7c6afc6a851c9", "input_tokens": 202, "server_ms": 35.07}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06981614232063, "probs": [0.0360284403416626, 0.9407485455412432, 0.023223014117094116], "top": "supported", "option_logits": null, "prompt_sha256": "09ba568fa9605e96b8b4411a9ca3aae662f9ea4803bb165ee522944d67fb01f7", "input_tokens": 196, "server_ms": 34.59}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40965900942683, "probs": [0.20906175007679842, 0.01029042356271627, 0.7806478263604854], "top": "insufficient", "option_logits": null, "prompt_sha256": "cf3bcade5ac0e68f64b9e3de92a3184b942d098eaacb9f24e3300b6b94598271", "input_tokens": 196, "server_ms": 34.91}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0598949957639, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.55}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18722704797983, "probs": [0.9052958620137371, 0.016330598826197848, 0.07837353916006504], "top": "contradicted", "option_logits": null, "prompt_sha256": "488e5a4d1a83bc993ce696396f30450bd15bb21481f5518dc951c048237b93f8", "input_tokens": 206, "server_ms": 34.71}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88954305835068, "probs": [0.03521557786284803, 0.04526073598298664, 0.9195236861541654], "top": "contradicted", "option_logits": null, "prompt_sha256": "fd494f61ffab19d399e8fb90311c05082df25926fb343185a8e2a5c0ef45865c", "input_tokens": 208, "server_ms": 34.39}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58065099827945, "probs": [0.956488422228046, 0.025140357850740234, 0.01837121992121384], "top": "insufficient", "option_logits": null, "prompt_sha256": "4556774746f89d946c8d41b92b4aadc4c1dd8ad7aec91836e3576a42b46e5d80", "input_tokens": 200, "server_ms": 35.11}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27571805939078, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 34.59}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09069499745965, "probs": [0.030254731557570292, 0.016155721873385053, 0.9535895465690447], "top": "supported", "option_logits": null, "prompt_sha256": "ef4c373b1ea1c02654d9f6861502d3e48140acc6e921812105a537a7306af012", "input_tokens": 194, "server_ms": 34.49}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.626811139285564, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "9a355bc7ff84249e36d0f77394943b3c2c2c55e930cfac4ab5ac607aff2f52ef", "input_tokens": 193, "server_ms": 33.96}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13912586681545, "probs": [0.06594123405239463, 0.9194290330499963, 0.014629732897608938], "top": "insufficient", "option_logits": null, "prompt_sha256": "802949693168d625f8a46ddd7cbf0664d1582312a4dfc3600136983f6f0cc525", "input_tokens": 192, "server_ms": 34.52}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80105204693973, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.22}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0053150691092, "probs": [0.934797668143608, 0.031578838206113354, 0.03362349365027861], "top": "contradicted", "option_logits": null, "prompt_sha256": "eb09cab56420a01e12c591b8cd5604aaae564c91d1eaca4f9f41ad0fd5b5233d", "input_tokens": 205, "server_ms": 34.37}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.58063989132643, "probs": [0.9662013384858644, 0.017429270244818, 0.01636939126931757], "top": "contradicted", "option_logits": null, "prompt_sha256": "ca7284b45d19cedd96649c7650fd2bc131a85aa7b67f347b312ffeddeb465811", "input_tokens": 199, "server_ms": 34.01}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25947695784271, "probs": [0.0494468893746347, 0.007071172681351224, 0.943481937944014], "top": "insufficient", "option_logits": null, "prompt_sha256": "0e8c3b5d6ee999770b09a8d41f121d17841a8a3c88f99ca31776d98e3cb9a396", "input_tokens": 194, "server_ms": 34.67}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68346116319299, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.09}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99217387475073, "probs": [0.03400516513017273, 0.94540886408499, 0.020585970784837124], "top": "supported", "option_logits": null, "prompt_sha256": "c091f0fb6754104f2838a339bd3f3b7c66ba89f33bb05f47107decd9d42301c7", "input_tokens": 198, "server_ms": 34.45}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.009445019066334, "probs": [0.03851217176175775, 0.9444509127063415, 0.017036915531900693], "top": "supported", "option_logits": null, "prompt_sha256": "498ed610492bb0a308366b9a8de1475f1d8a540b55af4127e4778ae21b45edbc", "input_tokens": 194, "server_ms": 34.33}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.306098034605384, "probs": [0.9707570026002149, 0.016446573157518886, 0.012796424242266244], "top": "insufficient", "option_logits": null, "prompt_sha256": "260057e3eb2abfe4ed09b701c6deff3fb3715fdb0c8ca324a1348dfd774f2c82", "input_tokens": 194, "server_ms": 34.69}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5688290912658, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.0}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91584407351911, "probs": [0.04850906601510146, 0.9255875820710483, 0.025903351913850195], "top": "contradicted", "option_logits": null, "prompt_sha256": "c9174dbf9bd60d78d5e86b6a48cf9c0e051be2112af8d047030597cc3117dbc8", "input_tokens": 211, "server_ms": 34.29}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7443619761616, "probs": [0.03386081007082737, 0.9413955327676918, 0.024743657161480793], "top": "contradicted", "option_logits": null, "prompt_sha256": "d475c03e8dd15fbc64354f2d0129fbd69e2b3ba4f1e325567f185dead7f95af1", "input_tokens": 201, "server_ms": 34.07}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29081712663174, "probs": [0.03868965206515757, 0.012507033178832465, 0.94880331475601], "top": "insufficient", "option_logits": null, "prompt_sha256": "6436470d3b6ae6205cb270b45c6c3d7ce3981366a138412b215b746633d85d03", "input_tokens": 197, "server_ms": 34.7}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.3971081096679, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 33.8}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89210303127766, "probs": [0.03448559198311794, 0.9587656533150248, 0.006748754701857216], "top": "permitted", "option_logits": null, "prompt_sha256": "2b94c94ac232ec104d6110952de80a76a00fe63134632daf52ba9abf7e6b6c98", "input_tokens": 218, "server_ms": 34.3}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.603521021083, "probs": [0.9014745524219904, 0.07329691914188313, 0.025228528436126444], "top": "permitted", "option_logits": null, "prompt_sha256": "dba7aef8f49c04c2fdddde2913d398830805d2a0db86474bf3980f131ff38f40", "input_tokens": 203, "server_ms": 34.84}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39786008372903, "probs": [0.07600853330506088, 0.046013876624721124, 0.8779775900702179], "top": "insufficient", "option_logits": null, "prompt_sha256": "ef6236452d6617250251cedd641ebfc6643ccf809d4ac681657e917ec058827e", "input_tokens": 209, "server_ms": 34.65}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.55827883630991, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 33.99}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88306291587651, "probs": [0.07865862910342072, 0.9085889611870406, 0.012752409709538734], "top": "prohibited", "option_logits": null, "prompt_sha256": "6877f3535caa9f30ae266f09910684ff8191f357e26f28235801e3ee61d863c6", "input_tokens": 214, "server_ms": 34.19}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70980199240148, "probs": [0.3226628352473404, 0.4415525480598312, 0.23578461669282835], "top": "insufficient", "option_logits": null, "prompt_sha256": "84f9c8de74a885791a9adb02311064234d0f3c511ab5d8bde62bc920f6c9ad41", "input_tokens": 209, "server_ms": 34.07}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94291296415031, "probs": [0.11063351808551876, 0.8770566410678413, 0.012309840846639953], "top": "insufficient", "option_logits": null, "prompt_sha256": "990c725e7e508f4293dd16353f1b16e7bb890bdc1bf5f3d13a693d4e617635c6", "input_tokens": 209, "server_ms": 34.31}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.4200877957046, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 33.86}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1903870254755, "probs": [0.05501866755885692, 0.07529107902609294, 0.8696902534150501], "top": "permitted", "option_logits": null, "prompt_sha256": "6e590e2135227d5cca28d2e60e24fd54e93c992a155bfc23179ee4c580915112", "input_tokens": 211, "server_ms": 34.36}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.877622827887535, "probs": [0.943076329528072, 0.040946062905879925, 0.01597760756604808], "top": "permitted", "option_logits": null, "prompt_sha256": "f0a55e8157d1588348f58d9d803f9bac80b18f2c0cd81e8afbf5f86cc0bc54fa", "input_tokens": 202, "server_ms": 34.21}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16083709150553, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "5fe8edf564c5caecc2816e1e5b24dc2e11e7e49716f15277d0ad87e04fb0441e", "input_tokens": 204, "server_ms": 34.57}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60645009949803, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.0}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.260767072439194, "probs": [0.08617432324783239, 0.8780222847407805, 0.03580339201138707], "top": "prohibited", "option_logits": null, "prompt_sha256": "de7b2b0f8c3b1dea2ae34d68854d8932985f8194c5d9ef94ef9ac3c5af3abf22", "input_tokens": 214, "server_ms": 34.67}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.54116904363036, "probs": [0.060153978502511585, 0.046803417311243246, 0.8930426041862451], "top": "prohibited", "option_logits": null, "prompt_sha256": "c58dd6040c3a5d9472ff44ab8e41e9eb92a7c0011e1b1ba9021afcf2d6b1a11a", "input_tokens": 213, "server_ms": 33.92}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37778915092349, "probs": [0.3370535895810829, 0.14003778663063773, 0.5229086237882794], "top": "insufficient", "option_logits": null, "prompt_sha256": "eb881b14aecfc2220563d1a2f324b20bccfcfe0fd127368ba2a8ed32ee653ac1", "input_tokens": 212, "server_ms": 34.76}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5557000041008, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 34.0}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04117611050606, "probs": [0.043034915318676176, 0.9309127048683882, 0.026052379812935716], "top": "permitted", "option_logits": null, "prompt_sha256": "ef8d68233fa0d3bb7526e3c22aa69a27e6f6fbda1b973de57cbe90e38379d4c0", "input_tokens": 216, "server_ms": 34.43}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.760490853339434, "probs": [0.9036036371977804, 0.039232255151330435, 0.057164107650889244], "top": "permitted", "option_logits": null, "prompt_sha256": "5f63161d15c1144a576bb01894090a0d08c4f3bfdcfe5fc99192ebe00be5a0ec", "input_tokens": 208, "server_ms": 34.11}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07210603170097, "probs": [0.8930317651393958, 0.06819498463159931, 0.03877325022900488], "top": "insufficient", "option_logits": null, "prompt_sha256": "98dbd799628876f678d1de46679eb443541ecb76f026ad2b8b781efa1705880a", "input_tokens": 215, "server_ms": 34.5}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.43857920542359, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 33.87}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77996201440692, "probs": [0.42551441767130554, 0.06085086611336375, 0.5136347162153307], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ae6464839efd58c0371c609fe15542b4468085854cec4634bb01da9bb661271", "input_tokens": 212, "server_ms": 34.24}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.787393037229776, "probs": [0.2363974954976129, 0.6450449475319231, 0.11855755697046397], "top": "prohibited", "option_logits": null, "prompt_sha256": "1ad4c75b16b0473861999a960bca9c7cce8eac322d72f239942517d07ecc7531", "input_tokens": 209, "server_ms": 34.14}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.850722040981054, "probs": [0.5169331333226727, 0.45597494786051085, 0.02709191881681649], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6546fb5bf3590e9fba39e6d78cfd36e6155363cfafeb691260ad754365b88a2", "input_tokens": 210, "server_ms": 34.23}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83683299459517, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 34.17}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89467301592231, "probs": [0.03143752450054959, 0.03794796085693116, 0.9306145146425192], "top": "B", "option_logits": null, "prompt_sha256": "d3eb956ec7e3e32cf66ce5d717522d8d92b5dac4c8308f4c9d4cb31fe756303e", "input_tokens": 201, "server_ms": 34.28}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92757384851575, "probs": [0.08136038144369948, 0.8826476192952379, 0.03599199926106259], "top": "B", "option_logits": null, "prompt_sha256": "cd399d53be08dd01e71ea966eaf17f42207db08d1eeeef6472c99f8f8fe53f13", "input_tokens": 192, "server_ms": 34.33}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.938313860446215, "probs": [0.10525990911259618, 0.888486034617855, 0.00625405626954877], "top": "A", "option_logits": null, "prompt_sha256": "eb3cee4bfe759b0bc88ef31833cbc8682cc73ee0d02c2a21d2d3ad10a8b33f3d", "input_tokens": 192, "server_ms": 34.28}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6720799356699, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.04}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.194617092609406, "probs": [0.04147910382575712, 0.95535340876032, 0.003167487413922894], "top": "A", "option_logits": null, "prompt_sha256": "6305741f94289ba3f3621c2b6eb0f190d9314fd35bf4428bc6faee6c58477958", "input_tokens": 211, "server_ms": 34.6}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.782251903787255, "probs": [0.9728143992091232, 0.0198945881954946, 0.007291012595382136], "top": "A", "option_logits": null, "prompt_sha256": "54e77d8b88f6d4d5ee3cfd402bbd6f306bf44e79427bd574d63cd3f4cfb05e72", "input_tokens": 200, "server_ms": 34.11}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21206705085933, "probs": [0.3414721146107966, 0.6394729578033058, 0.019054927585897746], "top": "A", "option_logits": null, "prompt_sha256": "5ace07508957ec619cb5e27ef2963d8fb638a1dbc11c9f5324862d847526978c", "input_tokens": 190, "server_ms": 34.49}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.054494954645634, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.35}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11610590107739, "probs": [0.019333349474436403, 0.0927842897505625, 0.887882360775001], "top": "B", "option_logits": null, "prompt_sha256": "0693c147f7e7d616b5187d9ab344a3da9ce0775cfe9c7bbbb763e22f35e065ba", "input_tokens": 206, "server_ms": 34.36}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.208705907687545, "probs": [0.03036360301441411, 0.9570210386636204, 0.012615358321965421], "top": "B", "option_logits": null, "prompt_sha256": "18217c60f50c0d632786b29cb4d780e7e97192fb8f872b0f1756a8ffa38a2edc", "input_tokens": 197, "server_ms": 34.32}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09879610687494, "probs": [0.1762660158495679, 0.1655472114938264, 0.6581867726566057], "top": "insufficient", "option_logits": null, "prompt_sha256": "688e414d6cdfa5a61e0f0668cb9967fd977b1f20a95cfe725e9bc169ed05549c", "input_tokens": 192, "server_ms": 34.42}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72012104652822, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.06}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36328799277544, "probs": [0.05076536678270011, 0.039498512654509606, 0.9097361205627902], "top": "A", "option_logits": null, "prompt_sha256": "882794cff48da2b9528a41f2219d35fab11c254f26ff72c815aa40552eb41e83", "input_tokens": 203, "server_ms": 34.61}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.948854103684425, "probs": [0.0494468893746347, 0.943481937944014, 0.007071172681351224], "top": "A", "option_logits": null, "prompt_sha256": "1325099ea764f8c6251ce0c006048d9276e68171e27006136ee3aa20f11a03e8", "input_tokens": 193, "server_ms": 34.23}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97139397263527, "probs": [0.8780235296813632, 0.10402040687475302, 0.01795606344388373], "top": "insufficient", "option_logits": null, "prompt_sha256": "a613883b12602a7fb71d9d55a911f2614490592581d6471116c4d4c68a4c8957", "input_tokens": 192, "server_ms": 34.3}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949103930965066, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 34.25}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88772302120924, "probs": [0.9647591205470982, 0.03259097850339806, 0.0026499009495037375], "top": "B", "option_logits": null, "prompt_sha256": "2b94950be2ada8957f3d093c3101d1935356aed904ff6f32af5e780ca4daf0fa", "input_tokens": 206, "server_ms": 34.19}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81749198585749, "probs": [0.900473990544217, 0.08837786536850668, 0.01114814408727632], "top": "B", "option_logits": null, "prompt_sha256": "98330242bf9b4853692144bd273028a1065e63fecbd536b99f72041a124cbdde", "input_tokens": 200, "server_ms": 34.13}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29564789682627, "probs": [0.46462686920294194, 0.07074626159411614, 0.46462686920294194], "top": "A", "option_logits": null, "prompt_sha256": "0a00b1c54151f720e7eb71eb04001a6783321bfab6008e2e1e9749ec127ea479", "input_tokens": 193, "server_ms": 34.6}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8319230619818, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.11}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.407558877021074, "probs": [0.05584400041528186, 0.9398915591084774, 0.004264440476240678], "top": "A", "option_logits": null, "prompt_sha256": "104d6ac93e38d37c2cbd2fe1d413e52ca92dde1258cac35fc220528b73196b21", "input_tokens": 209, "server_ms": 34.74}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.923523992300034, "probs": [0.04937458683512937, 0.008523069447562875, 0.9421023437173077], "top": "A", "option_logits": null, "prompt_sha256": "d217d38b7e6213bb691099e21c66a3df1e10fe6cde75e36ed398dd45fd11b8e9", "input_tokens": 194, "server_ms": 34.19}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.300379015505314, "probs": [0.19870361005143694, 0.18662038312341964, 0.6146760068251435], "top": "insufficient", "option_logits": null, "prompt_sha256": "ee048efb2e813aef04a41f981140fbeb7ab7de14a0a4baac09c4a808bb63510f", "input_tokens": 192, "server_ms": 34.39}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9598139077425, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 34.19}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.141626002267, "probs": [0.8170038444540815, 0.05859532327955457, 0.12440083226636395], "top": "supported", "option_logits": null, "prompt_sha256": "17fe8aeab450890957725a1dad43750752505b560e3976c468dc3f41172f8a67", "input_tokens": 214, "server_ms": 34.43}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.60214017145336, "probs": [0.015242642754936796, 0.026808995604051852, 0.9579483616410114], "top": "supported", "option_logits": null, "prompt_sha256": "9e0440adb82ccd063e1c322dc6a4b4897225905d32a3f7f1bb9049191611cc67", "input_tokens": 209, "server_ms": 33.86}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31788811273873, "probs": [0.05587209717642955, 0.9403644443692992, 0.0037634584542711896], "top": "insufficient", "option_logits": null, "prompt_sha256": "d0c71930cb8f313e647a558d26ffefabb510d65a93a3f32e41872606cc6bf95b", "input_tokens": 207, "server_ms": 34.46}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73789091408253, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.02}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.142295856028795, "probs": [0.28553873616003933, 0.3237117094730352, 0.39074955436692543], "top": "permitted", "option_logits": null, "prompt_sha256": "daa6ab1802b1fde61e5a315b23c0a03e284c6fadfbc803e995b1b2f126db4ff1", "input_tokens": 229, "server_ms": 34.47}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73773095943034, "probs": [0.9139932977706335, 0.05100292527189731, 0.035003776957469146], "top": "permitted", "option_logits": null, "prompt_sha256": "bcfabc366ceb030ad33c3667e455143117c067135419a04c514f9eedf56beef4", "input_tokens": 222, "server_ms": 34.04}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85357398353517, "probs": [0.055084494859207855, 0.017806923046620855, 0.9271085820941714], "top": "insufficient", "option_logits": null, "prompt_sha256": "4648f72540215e378c489522ed8bd6dc3c3d325e632d3d70fb9169a60a51ca09", "input_tokens": 221, "server_ms": 34.13}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86351305805147, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 34.2}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06119488924742, "probs": [0.04144166000042449, 0.9544910095891707, 0.004067330410404966], "top": "A", "option_logits": null, "prompt_sha256": "2298533e13693dfd7fd2fbb3c3e005385ef1d63a9b81983a9d3e25feb90ff23c", "input_tokens": 210, "server_ms": 34.39}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0428750757128, "probs": [0.6989250414551824, 0.06050754282191963, 0.240567415722898], "top": "A", "option_logits": null, "prompt_sha256": "c9d6f93bbd17ec079e798d376df024790d2bfa77871b44e70fd8616a0ad58211", "input_tokens": 209, "server_ms": 34.26}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69321204163134, "probs": [0.9290594611076342, 0.055200406122332214, 0.015740132770033617], "top": "insufficient", "option_logits": null, "prompt_sha256": "e508f7c0cf5b6f7bf721fbea254629adeb5723dd71b126c9dad2526fd6478e69", "input_tokens": 211, "server_ms": 34.91}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2435268945992, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 34.56}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2595579829067, "probs": [0.7687919368693059, 0.037841427439577066, 0.19336663569111703], "top": "supported", "option_logits": null, "prompt_sha256": "2576bc0fe1d0394c448cc1d2e3cd03c33f1d7a6a8b5f8c0c267ac79d4ecbb2e0", "input_tokens": 227, "server_ms": 34.53}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.323158932849765, "probs": [0.028437503812587046, 0.01721543167027419, 0.9543470645171389], "top": "supported", "option_logits": null, "prompt_sha256": "c88d87385c3f5978b7b5ff8266b180fb20725224ea5035ed6f7e759ad7175b45", "input_tokens": 202, "server_ms": 34.6}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00166521593928, "probs": [0.03842465563368609, 0.01927064844887017, 0.9423046959174438], "top": "insufficient", "option_logits": null, "prompt_sha256": "5d00998ccf948a378fe3730d22365cc93d6ca3447a6bffab9c27db9ba203c73f", "input_tokens": 202, "server_ms": 34.31}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77537199109793, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.07}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20243601128459, "probs": [0.4756119457847697, 0.27041607197430173, 0.2539719822409286], "top": "insufficient", "option_logits": null, "prompt_sha256": "f26149405c562e81cb9ef93186374e44a3d38ec943d4bc1328305de7bfd2f3c5", "input_tokens": 231, "server_ms": 34.41}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.61660011857748, "probs": [0.8133507124046239, 0.05478605406972841, 0.13186323352564766], "top": "permitted", "option_logits": null, "prompt_sha256": "bcca8c96798eb11f3f4644bd6b48d9956d0ef797acb354a9d313e72d7decc3e6", "input_tokens": 216, "server_ms": 33.89}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52113110385835, "probs": [0.17523657347175897, 0.6967099745500006, 0.1280534519782404], "top": "insufficient", "option_logits": null, "prompt_sha256": "39afa3600a44ace025f89dc8a73ad8b6ad0761af51bdcb709832b17c14127f54", "input_tokens": 223, "server_ms": 34.68}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.364028161391616, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 34.64}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51251986250281, "probs": [0.9147597146359207, 0.07919284608441045, 0.006047439279669017], "top": "B", "option_logits": null, "prompt_sha256": "c4b6e37f55e9e861c7a6f927859de07ee371fe7aca51ca2ae85600e1cd82765e", "input_tokens": 222, "server_ms": 34.81}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.144616013392806, "probs": [0.8768319468998427, 0.09162951349252214, 0.03153853960763513], "top": "B", "option_logits": null, "prompt_sha256": "83a38a50ea29d93de3d3e3463c307552985b65ef7fbecfb33f19f0c8b19cbe78", "input_tokens": 215, "server_ms": 34.29}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62122197076678, "probs": [0.9751721172869884, 0.022608910103960853, 0.0022189726090507235], "top": "insufficient", "option_logits": null, "prompt_sha256": "65c35f2d10deb3715357e389ebf9ee5a130cde3069dda04db44dcb11491b8adc", "input_tokens": 201, "server_ms": 34.92}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.491270108148456, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.56}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27312805131078, "probs": [0.08075843891914115, 0.04312419313814581, 0.876117367942713], "top": "supported", "option_logits": null, "prompt_sha256": "4f8bbbb0dfd533b6deedfbbd1466a23c3b560303a1f961ed21be386fa842301a", "input_tokens": 219, "server_ms": 34.41}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77243203856051, "probs": [0.9535895465690447, 0.016155721873385053, 0.030254731557570292], "top": "supported", "option_logits": null, "prompt_sha256": "c61081105c19ff493658ba57f096ae6e9380cb6cb3a7cdc1596f4c36991b4710", "input_tokens": 220, "server_ms": 34.02}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38697990775108, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "insufficient", "option_logits": null, "prompt_sha256": "94d9b1e865081ca9ee6a9204c96cb1afbd9655d3298d6fd5bdca85429f6d7531", "input_tokens": 218, "server_ms": 34.52}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.847144132480025, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.16}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11879579350352, "probs": [0.47103444559681956, 0.08657488347540951, 0.442390670927771], "top": "prohibited", "option_logits": null, "prompt_sha256": "0984ef33039be3140b3da26a6910b546b1f605153e5b197341ffac510f80627a", "input_tokens": 221, "server_ms": 34.42}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9371441192925, "probs": [0.049073786622505555, 0.0715038981480174, 0.879422315229477], "top": "permitted", "option_logits": null, "prompt_sha256": "cd350bbed40603bfdec8aee19ed8f4aead08aae6c3e75e5950990320c9a1602f", "input_tokens": 221, "server_ms": 34.24}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13867696933448, "probs": [0.039087581768329945, 0.9002715143788037, 0.06064090385286648], "top": "insufficient", "option_logits": null, "prompt_sha256": "32a2ba65d112e44959b5167505e017413199fda0540cd2a995538aa795dabd11", "input_tokens": 220, "server_ms": 34.34}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80548211559653, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.01}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00097487308085, "probs": [0.02394587818769938, 0.9700312518040514, 0.006022870008249252], "top": "B", "option_logits": null, "prompt_sha256": "0d78a5511291c0e8891da715a82ec2ad438e94f5dddaee4ff0a75c528e00ad7e", "input_tokens": 230, "server_ms": 34.29}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53215989470482, "probs": [0.028667490838461873, 0.9620652951327192, 0.00926721402881876], "top": "B", "option_logits": null, "prompt_sha256": "f925b35d49f8ca5aac7190490beee3fe1f89e7ea2183f46b6ff8de5291cc99d6", "input_tokens": 223, "server_ms": 33.81}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.053073989227414, "probs": [0.018723542031095865, 0.9748318841721378, 0.0064445737967663164], "top": "insufficient", "option_logits": null, "prompt_sha256": "6c61299dde52e0bf1bae5a513eccf6464ab11fd42eab81e0c811d717c5ee78fe", "input_tokens": 210, "server_ms": 34.34}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76482196897268, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.01}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.223846884444356, "probs": [0.8748952051681804, 0.027757979619760474, 0.09734681521205904], "top": "supported", "option_logits": null, "prompt_sha256": "b21acb3da5a82760fd02a24aed0fc2babe9ffe3bc13c3b214a041901cb3e0d7d", "input_tokens": 224, "server_ms": 34.52}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69260093010962, "probs": [0.07897871706298064, 0.6261104979730077, 0.29491078496401174], "top": "insufficient", "option_logits": null, "prompt_sha256": "11a658148b97ed6304e23da68b8382fe860758bceb2aab62101a22fb8e2a2d9b", "input_tokens": 220, "server_ms": 33.99}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.100394954904914, "probs": [0.9241406947470572, 0.024290130010671554, 0.051569175242271295], "top": "insufficient", "option_logits": null, "prompt_sha256": "d6185ec81e36298077e1827a7b4a5e68c397cf8dde6ebe5e37e25b43ca01882d", "input_tokens": 208, "server_ms": 34.32}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66844079270959, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 33.96}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.124786991626024, "probs": [0.21207317204418882, 0.04418942927644211, 0.743737398679369], "top": "insufficient", "option_logits": null, "prompt_sha256": "529e850d9f1fbc6eacd215e95ebfbc941317a3f5224bff5d7dea44065cd2e1a9", "input_tokens": 228, "server_ms": 34.44}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.621090956032276, "probs": [0.045449474437266706, 0.031192392974493464, 0.9233581325882398], "top": "permitted", "option_logits": null, "prompt_sha256": "81df99f7f22c5619613f932264037a494407bd0c9c1cd772d88cdb13bd93f038", "input_tokens": 225, "server_ms": 33.94}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19173604622483, "probs": [0.9572108985191267, 0.032335988189970806, 0.010453113290902514], "top": "insufficient", "option_logits": null, "prompt_sha256": "737e84a1e8c9bcc630c5ec500de2a3e5db92ca702b06dcd3ad924711e2fbdbc3", "input_tokens": 218, "server_ms": 34.52}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89968399703503, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.13}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85550298541784, "probs": [0.37235037560509904, 0.615071082756213, 0.012578541638687917], "top": "A", "option_logits": null, "prompt_sha256": "c5cb029101b9f71556e6e018776a50195e4ada1ad92940e55665671b3868297a", "input_tokens": 227, "server_ms": 34.18}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94245405495167, "probs": [0.12177174509735464, 0.07849096626584667, 0.7997372886367987], "top": "A", "option_logits": null, "prompt_sha256": "dd57bc49c9dda9182642dd072b29b16671a713f0b0824c487439f9e2b331573e", "input_tokens": 207, "server_ms": 34.21}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02557506039739, "probs": [0.073881614277135, 0.017452691866116716, 0.9086656938567483], "top": "insufficient", "option_logits": null, "prompt_sha256": "bda0e38b4228f4220bca836f2281dd0b297cef74516258475f031ebc54637c78", "input_tokens": 197, "server_ms": 34.33}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.734772147610784, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 34.04}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.104694871231914, "probs": [0.9825585277035805, 0.01007742973075826, 0.007364042565661154], "top": "supported", "option_logits": null, "prompt_sha256": "cc5454a07757b53ce2f838d93f3f4c26856cbc40d4fdb7c2edb21f43da36a5cc", "input_tokens": 212, "server_ms": 34.4}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.729032173752785, "probs": [0.02825825432560628, 0.023410205377014614, 0.9483315402973791], "top": "supported", "option_logits": null, "prompt_sha256": "b1739f17dac9c6d981d59ce4523ac7852dffdb509918039bd8d833acb430ac6c", "input_tokens": 202, "server_ms": 34.0}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13227598927915, "probs": [0.6892396519371748, 0.005856275914839899, 0.30490407214798526], "top": "insufficient", "option_logits": null, "prompt_sha256": "98f4588180d28f5dcef4c3bf07ecf8e021f406ba222ff13a774d337df06379ee", "input_tokens": 203, "server_ms": 34.39}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68167092837393, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 33.95}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15205704793334, "probs": [0.036422612390539545, 0.9510408592905502, 0.012536528318910257], "top": "permitted", "option_logits": null, "prompt_sha256": "1e8882942aa8fcb63c49bda41019547d5c388d1eb722d8f3d2f6ff0fe2f6de83", "input_tokens": 221, "server_ms": 34.41}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8096421007067, "probs": [0.03221192217085971, 0.014249827577513809, 0.9535382502516264], "top": "permitted", "option_logits": null, "prompt_sha256": "7ab28da3bc25386c1a377fd6ee20ae9c0e59159093b9600f9039749931b1d405", "input_tokens": 215, "server_ms": 34.09}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01395494863391, "probs": [0.6096504219680668, 0.3466255945273742, 0.043723983504558944], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce7c09276f0384e8534c52aa20c9df30f1b7a23977287fb0373141421ede2323", "input_tokens": 220, "server_ms": 34.29}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65500087663531, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 33.96}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06662496551871, "probs": [0.026052379812935716, 0.043034915318676176, 0.9309127048683882], "top": "B", "option_logits": null, "prompt_sha256": "3e92d9a3ec71acb771522ae129c59a1c3f3cf135471b1a9119034e8e60f57534", "input_tokens": 212, "server_ms": 34.33}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.783261923119426, "probs": [0.010960515985196186, 0.046398608171146855, 0.942640875843657], "top": "B", "option_logits": null, "prompt_sha256": "6c5f599d35035dfb7793d063356814cf6c7a52ffa3dd47f92c4bf500ca7dfd33", "input_tokens": 203, "server_ms": 34.07}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16854588128626, "probs": [0.039029590587771126, 0.9571398140015086, 0.0038305954107202447], "top": "insufficient", "option_logits": null, "prompt_sha256": "37218ff944f5d39cce6127759ca5add34909346542a0085ae71b27113a5d94dc", "input_tokens": 191, "server_ms": 34.44}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.804871866479516, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.1}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07390488125384, "probs": [0.06296298480307037, 0.877902829429632, 0.05913418576729761], "top": "supported", "option_logits": null, "prompt_sha256": "ffb22e022a23a832611c31cce83e5ded9325f32d02e4447965afe9987da2bde6", "input_tokens": 221, "server_ms": 34.33}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.841542925685644, "probs": [0.9514443415619939, 0.009758312735022333, 0.038797345702983814], "top": "supported", "option_logits": null, "prompt_sha256": "7a3eec9452b83ccbec55aa1300b3feef68ed5e14955200c3b0e7ea8e89a95d42", "input_tokens": 229, "server_ms": 34.18}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11158502846956, "probs": [0.026899996114941614, 0.011899951244465976, 0.9612000526405924], "top": "insufficient", "option_logits": null, "prompt_sha256": "6210dc877e4c1f0e5a92c78000354ff443f3add66bc382cd9fc3184bc1cde515", "input_tokens": 226, "server_ms": 34.4}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.816943204030395, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.08}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.225716980174184, "probs": [0.12359582333016605, 0.8642737486438103, 0.01213042802602361], "top": "permitted", "option_logits": null, "prompt_sha256": "b064243d3870600f856ee53a1ef805a900c11107d12f38f2c3670753f532aff4", "input_tokens": 215, "server_ms": 34.5}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.611270159482956, "probs": [0.20386035032714242, 0.5922792993457151, 0.20386035032714242], "top": "permitted", "option_logits": null, "prompt_sha256": "d0e6877a0a8a83928384752e861479053fa3570f13d19a5a7be7d8ed7368055d", "input_tokens": 225, "server_ms": 33.92}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12106612510979, "probs": [0.4345988057387763, 0.10266314506169773, 0.462738049199526], "top": "insufficient", "option_logits": null, "prompt_sha256": "f8d2b6eea2aa49d96d8da956f53d998cbb5844a3b8354672ea7172d7f3564f75", "input_tokens": 224, "server_ms": 34.39}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.699420074000955, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 33.97}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03656489960849, "probs": [0.15827071505004672, 0.808750606384142, 0.032978678565811305], "top": "B", "option_logits": null, "prompt_sha256": "d0cfebf59620575e28c1572d9cd5fe9a5f21b04d6f113348f66e043ebd3db047", "input_tokens": 218, "server_ms": 34.28}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.61108994856477, "probs": [0.8012135216734096, 0.1567957257725581, 0.04199075255403233], "top": "B", "option_logits": null, "prompt_sha256": "7cb2185c68200e9ee262ca2de7508a98cc4dcde47a5db7a8e9587bc9489b12cf", "input_tokens": 209, "server_ms": 33.88}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00698493048549, "probs": [0.06962296989463873, 0.018645411826949065, 0.9117316182784122], "top": "insufficient", "option_logits": null, "prompt_sha256": "cfa1a906fceb6b3b9d7b5c051d4c802da2bb13a38396312d0a36f9d579150568", "input_tokens": 207, "server_ms": 34.26}], "seconds": 5.3, "failures": 0}, "shuffled": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.745040910318494, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 33.97}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.257477873936296, "probs": [0.9674618846083878, 0.012752988686722479, 0.01978512670488969], "top": "supported", "option_logits": null, "prompt_sha256": "eec32c94bdef649fc890b55ec49cf1aa2f19a0462a3385eb89ed703a10ecac10", "input_tokens": 202, "server_ms": 34.48}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78700211457908, "probs": [0.03821459235254428, 0.024632151448974626, 0.9371532561984811], "top": "supported", "option_logits": null, "prompt_sha256": "48ceb67afca89ff6de3c590c914dcb0fd4b9e8e4348d19e33d11bcddfee33702", "input_tokens": 196, "server_ms": 34.01}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42330987006426, "probs": [0.20906175007679842, 0.01029042356271627, 0.7806478263604854], "top": "insufficient", "option_logits": null, "prompt_sha256": "cf3bcade5ac0e68f64b9e3de92a3184b942d098eaacb9f24e3300b6b94598271", "input_tokens": 196, "server_ms": 34.67}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.979263881221414, "probs": [0.028546574507428056, 0.01344601762615991, 0.958007407866412], "top": "supported", "option_logits": null, "prompt_sha256": "55d43d0de49c72c7c509fc43c61179257e038a6cfe16e115f6a2a0e587b33824", "input_tokens": 209, "server_ms": 34.14}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.792851984500885, "probs": [0.07810937869521252, 0.9022445396051365, 0.019646081699650965], "top": "contradicted", "option_logits": null, "prompt_sha256": "b45f2030e55f3fdf7179b1673f65e09b09bd2a5659a186837a6bb62fbd78bd00", "input_tokens": 206, "server_ms": 34.1}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.72243095934391, "probs": [0.0464300360263431, 0.8859181944824366, 0.06765176949122025], "top": "contradicted", "option_logits": null, "prompt_sha256": "223a68676447bbcb6017f47a40a96cd18bed0fa7f075b38653f938db60b3baa9", "input_tokens": 208, "server_ms": 33.95}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13331604376435, "probs": [0.9436516791652002, 0.02640889059428092, 0.02993943024051884], "top": "insufficient", "option_logits": null, "prompt_sha256": "24d025485f5e4a2bff769117ed138515cf93200b06e742e455f7b8cc7152e3de", "input_tokens": 200, "server_ms": 34.42}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.56554301455617, "probs": [0.9674618846083878, 0.01978512670488969, 0.012752988686722479], "top": "contradicted", "option_logits": null, "prompt_sha256": "b372c17c6c8de4fafff4d6c0ce468d3b54126dbfeef46d08db082bc829064884", "input_tokens": 194, "server_ms": 35.32}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.040400974452496, "probs": [0.02369913709056687, 0.016264936907900464, 0.9600359260015328], "top": "supported", "option_logits": null, "prompt_sha256": "57e9f770948d7778575f35784aa034b97ae3b3b2f2bd934c9e75f5fe7441659a", "input_tokens": 194, "server_ms": 37.86}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.1102391295135, "probs": [0.9757912442213938, 0.012862785395550868, 0.011345970383055229], "top": "supported", "option_logits": null, "prompt_sha256": "61b6f283623ce18a30fbb0b6e030082367529d979b50606a19e5c27e971820ee", "input_tokens": 193, "server_ms": 36.29}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.487221114337444, "probs": [0.055144207653558895, 0.01674220672178504, 0.928113585624656], "top": "insufficient", "option_logits": null, "prompt_sha256": "7cff73a57c8f079413f2a0b18374068ed86d94e2fafc91677c3742c14a066cab", "input_tokens": 192, "server_ms": 35.72}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.861473858356476, "probs": [0.028650300519022835, 0.009861327530892165, 0.9614883719500851], "top": "supported", "option_logits": null, "prompt_sha256": "7c45f0c0139d61a53e6208f0099673275733ff83d3f75da4dc4c78f43036b0f8", "input_tokens": 205, "server_ms": 35.11}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8051339648664, "probs": [0.02907704833321378, 0.05445242977074118, 0.916470521896045], "top": "contradicted", "option_logits": null, "prompt_sha256": "50848dffd2c1c615f5b75f0bb9bd41d494049c38a0e68f53f8fb55948e5b8111", "input_tokens": 205, "server_ms": 35.11}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65201196074486, "probs": [0.028121917950149154, 0.9437561640997016, 0.028121917950149154], "top": "contradicted", "option_logits": null, "prompt_sha256": "d9a00b5a305e35828cb17b95e58a1533b2089ed424d7bffc91ecfce3160c9611", "input_tokens": 199, "server_ms": 34.84}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.653871811926365, "probs": [0.0494468893746347, 0.007071172681351224, 0.943481937944014], "top": "insufficient", "option_logits": null, "prompt_sha256": "0e8c3b5d6ee999770b09a8d41f121d17841a8a3c88f99ca31776d98e3cb9a396", "input_tokens": 194, "server_ms": 34.97}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55460097454488, "probs": [0.1014573935767339, 0.042153146641024326, 0.8563894597822418], "top": "contradicted", "option_logits": null, "prompt_sha256": "e924e4a25b3371462e128b92799f79eea841a96f2a2b5ef2d8d204f809b5e06b", "input_tokens": 198, "server_ms": 34.78}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.21156802587211, "probs": [0.02825825432560628, 0.023410205377014614, 0.9483315402973791], "top": "supported", "option_logits": null, "prompt_sha256": "08da99ccc2e188d94eb1dd1b284d80a2f94349d15043417dadbf3ea7bb569d06", "input_tokens": 198, "server_ms": 35.38}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34476794488728, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "supported", "option_logits": null, "prompt_sha256": "8408e99d3352a39e8e873a1c4cef9a62a8418f37bdff3e002eb1f6f2306fa17b", "input_tokens": 194, "server_ms": 34.54}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.09156687930226, "probs": [0.9707570026002149, 0.016446573157518886, 0.012796424242266244], "top": "insufficient", "option_logits": null, "prompt_sha256": "260057e3eb2abfe4ed09b701c6deff3fb3715fdb0c8ca324a1348dfd774f2c82", "input_tokens": 194, "server_ms": 35.26}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50998014584184, "probs": [0.027692497402509345, 0.04296244408302007, 0.9293450585144706], "top": "supported", "option_logits": null, "prompt_sha256": "4b7f6e13d64e3b44753244fe8bcc35e3c61bb5cbc56885204150c2b8a1d730f4", "input_tokens": 211, "server_ms": 34.66}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.571900989860296, "probs": [0.038153743171136366, 0.026185265088641236, 0.9356609917402224], "top": "contradicted", "option_logits": null, "prompt_sha256": "a8983f75a788c808d83e49e31b2c7fc60f168203e3371a2f46e674b96cba586a", "input_tokens": 211, "server_ms": 34.73}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20121691003442, "probs": [0.0382719206436522, 0.023168966945696274, 0.9385591124106515], "top": "contradicted", "option_logits": null, "prompt_sha256": "b0a3a7311091e1405f7995266a413ad483489714075309b06ec871e31f77954f", "input_tokens": 201, "server_ms": 34.34}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46836096979678, "probs": [0.03621509737243083, 0.9456223856962459, 0.01816251693132339], "top": "insufficient", "option_logits": null, "prompt_sha256": "da3839c491c29e9cc50a18a3a20d59e3f9ff7b0e96a001ae41c4bf65f5f0eab1", "input_tokens": 197, "server_ms": 34.63}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05691499635577, "probs": [0.6993354607136348, 0.1872858543621493, 0.11337868492421596], "top": "prohibited", "option_logits": null, "prompt_sha256": "b4ef5af7abdfbfcdcb84a50cd028b197644864319b662c102080599d74c2a232", "input_tokens": 212, "server_ms": 34.22}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43007895536721, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "permitted", "option_logits": null, "prompt_sha256": "b961f1c42a0a57ea80e203f2f69b241844174acdf3b8cfe84ce6ec3ffb79fc8c", "input_tokens": 218, "server_ms": 34.62}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12374693714082, "probs": [0.05035348378460727, 0.9023550434611196, 0.047291472754273096], "top": "permitted", "option_logits": null, "prompt_sha256": "742a7db029a78d04d26d019231b01690c69683a73384a8da6aa929cd15c81513", "input_tokens": 203, "server_ms": 34.23}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.152237026020885, "probs": [0.09241428693559126, 0.8843416612729131, 0.023244051791495635], "top": "insufficient", "option_logits": null, "prompt_sha256": "094c6ca6168a09f4dc1447959e11a10d6a06018c0e4cfebb716806ee019cbfc0", "input_tokens": 209, "server_ms": 34.39}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8456628639251, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.13}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0592949911952, "probs": [0.9137406766907489, 0.06977638840966881, 0.016482934899582382], "top": "prohibited", "option_logits": null, "prompt_sha256": "c831b10f220597e08250d09cef04d9dcbbd20d23498389b2ec05cf891745d2e4", "input_tokens": 214, "server_ms": 34.27}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96158388629556, "probs": [0.3226628352473404, 0.4415525480598312, 0.23578461669282835], "top": "insufficient", "option_logits": null, "prompt_sha256": "84f9c8de74a885791a9adb02311064234d0f3c511ab5d8bde62bc920f6c9ad41", "input_tokens": 209, "server_ms": 34.22}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83947306126356, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "171ddb6e8d557e07c9968a706d339748b14038c7411892826fa822fb48e1baf6", "input_tokens": 209, "server_ms": 34.16}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75583191215992, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.03}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.145506124943495, "probs": [0.05501866755885692, 0.07529107902609294, 0.8696902534150501], "top": "permitted", "option_logits": null, "prompt_sha256": "6e590e2135227d5cca28d2e60e24fd54e93c992a155bfc23179ee4c580915112", "input_tokens": 211, "server_ms": 34.44}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68344090692699, "probs": [0.048789556914816484, 0.020270905643033604, 0.93093953744215], "top": "permitted", "option_logits": null, "prompt_sha256": "3f39f09c7d9e21ffc1f78839202c597a69077efd392d449211c28f002576f82d", "input_tokens": 202, "server_ms": 33.98}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17456595040858, "probs": [0.21264458466138933, 0.7457413364088059, 0.04161407892980468], "top": "insufficient", "option_logits": null, "prompt_sha256": "faac46392097109effab43e6f29e0f0866a09da218a45f79f5778c2e5a65a73d", "input_tokens": 204, "server_ms": 34.47}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90245398320258, "probs": [0.058304340167377615, 0.02006813943899786, 0.9216275203936245], "top": "permitted", "option_logits": null, "prompt_sha256": "dbea7157a3c37b6742682cae5b072b1054dd3ad36476b8a23466297be6d5f638", "input_tokens": 214, "server_ms": 34.17}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20833803527057, "probs": [0.08617432324783239, 0.8780222847407805, 0.03580339201138707], "top": "prohibited", "option_logits": null, "prompt_sha256": "de7b2b0f8c3b1dea2ae34d68854d8932985f8194c5d9ef94ef9ac3c5af3abf22", "input_tokens": 214, "server_ms": 34.4}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90700395964086, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "1aff6b562ec348487995092a4cf68d351f0a76eea07f42c0842730b8f60281f4", "input_tokens": 213, "server_ms": 34.1}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.265256978571415, "probs": [0.13537010767547805, 0.41875814284681956, 0.4458717494777024], "top": "insufficient", "option_logits": null, "prompt_sha256": "a0ea731df0f9f147afafd6706401757982b609f60c79d64c03ebfea463ef9821", "input_tokens": 212, "server_ms": 34.51}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.892833886668086, "probs": [0.06461138798541208, 0.9008867824895763, 0.034501829525011594], "top": "prohibited", "option_logits": null, "prompt_sha256": "dd3955a315dde371dc5e4ce9792fa9390e46cc33b9d2c6592ffb03bce46f1738", "input_tokens": 216, "server_ms": 34.06}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10253513418138, "probs": [0.043034915318676176, 0.9309127048683882, 0.026052379812935716], "top": "permitted", "option_logits": null, "prompt_sha256": "ef8d68233fa0d3bb7526e3c22aa69a27e6f6fbda1b973de57cbe90e38379d4c0", "input_tokens": 216, "server_ms": 34.36}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62605001591146, "probs": [0.08884445347431193, 0.060974765390393336, 0.8501807811352947], "top": "permitted", "option_logits": null, "prompt_sha256": "72458905854c2bd8458210955b878067b9d46d1b7595a462d19f98f25c4fbfa2", "input_tokens": 208, "server_ms": 33.9}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.109465803951025, "probs": [0.21015003684890857, 0.05285696229504507, 0.7369930008560464], "top": "insufficient", "option_logits": null, "prompt_sha256": "1eb890e1e9003931d9ddcfe01bc6929ca77211b8a8ab9237c546de8c3001faa3", "input_tokens": 215, "server_ms": 34.44}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91044286824763, "probs": [0.11741505362023608, 0.2826034863954738, 0.5999814599842902], "top": "permitted", "option_logits": null, "prompt_sha256": "5167011ba9be4d3d5e4675e3946f7775214985b6acc67629d42da678b376b719", "input_tokens": 209, "server_ms": 34.13}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.186007015407085, "probs": [0.42551441767130554, 0.06085086611336375, 0.5136347162153307], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ae6464839efd58c0371c609fe15542b4468085854cec4634bb01da9bb661271", "input_tokens": 212, "server_ms": 34.32}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.104385904967785, "probs": [0.29549446261685514, 0.11530520829956423, 0.5892003290835807], "top": "prohibited", "option_logits": null, "prompt_sha256": "9231010e808cb51e90e91e319911586ed3aeb9b5c6d6c648c044df8b8287b77e", "input_tokens": 209, "server_ms": 34.17}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04617405310273, "probs": [0.5169331333226727, 0.45597494786051085, 0.02709191881681649], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6546fb5bf3590e9fba39e6d78cfd36e6155363cfafeb691260ad754365b88a2", "input_tokens": 210, "server_ms": 34.28}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.916573997586966, "probs": [0.028170092258100337, 0.026457059155058287, 0.9453728485868413], "top": "A", "option_logits": null, "prompt_sha256": "2c3c3a83be0ee81ef6d6592eccaec7e6a6ba5b2ea0d6391d9d86d5e775337c59", "input_tokens": 200, "server_ms": 34.14}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06102492287755, "probs": [0.032411181205817395, 0.9594367541920572, 0.008152064602125513], "top": "B", "option_logits": null, "prompt_sha256": "91903b178a53be97c6b4e21b1d0650ef3e043300c347ae23ae561a14234e3d7e", "input_tokens": 201, "server_ms": 34.35}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93174407817423, "probs": [0.9231307043235732, 0.06218066343701687, 0.014688632239409829], "top": "B", "option_logits": null, "prompt_sha256": "3dff5c64bbce09d87dfb74fabb20e7031a8984b18f71249b72bfd902a583b649", "input_tokens": 192, "server_ms": 34.11}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06565499305725, "probs": [0.04181158538259194, 0.05373823788322395, 0.904450176734184], "top": "A", "option_logits": null, "prompt_sha256": "cf4e23a2812a6cdaf0dfd19302a5eda22c6153968a2782074212b2be313cfc91", "input_tokens": 192, "server_ms": 34.3}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.907452857121825, "probs": [0.9579542284336563, 0.039062801461222305, 0.0029829701051215123], "top": "B", "option_logits": null, "prompt_sha256": "29a2165d171c923e1807421722f38341ede9d1fb3e93a0c5373c0d3ba55ae49a", "input_tokens": 208, "server_ms": 34.11}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97335394099355, "probs": [0.9780535330693921, 0.014616188125699304, 0.007330278804908576], "top": "A", "option_logits": null, "prompt_sha256": "21515e5ecf0e10fc73cbe4c24cbbd88c4224cae289a96d2b0e594ae3597b31e8", "input_tokens": 211, "server_ms": 34.33}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84842283837497, "probs": [0.9618961177282845, 0.02691947679068596, 0.011184405481029442], "top": "A", "option_logits": null, "prompt_sha256": "6ce513c832609614a5c86a1da793e3dc9479da4bd0d08df92ef7858289130727", "input_tokens": 200, "server_ms": 34.1}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27934812195599, "probs": [0.17805852075193587, 0.15706137389923117, 0.664880105348833], "top": "A", "option_logits": null, "prompt_sha256": "a496e0a84ecb3fab0bcbcbf67c42d43175ca71241f413bb499c2d25cf5b89415", "input_tokens": 190, "server_ms": 34.52}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.875572986900806, "probs": [0.07003258404452323, 0.012871804091497973, 0.9170956118639788], "top": "A", "option_logits": null, "prompt_sha256": "601893abb138ce73b2b6942d4c60e2602f97eb9895db7c93790ee0080d24b941", "input_tokens": 203, "server_ms": 34.13}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25436795502901, "probs": [0.03871909899223181, 0.9495254842123682, 0.011755416795400006], "top": "B", "option_logits": null, "prompt_sha256": "d3cf5c157fb76c1b82e4fd00b70aad0c853e81926e95dc24dca56c08cb0094d2", "input_tokens": 206, "server_ms": 34.44}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.121966015547514, "probs": [0.028505129122276228, 0.07305053773408933, 0.8984443331436344], "top": "B", "option_logits": null, "prompt_sha256": "cc8fa7a50f3be03bcd0afd14f9ec758a2e5a1b0b7ef163beb146ecabb71b14ee", "input_tokens": 197, "server_ms": 34.42}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08992595784366, "probs": [0.7064924652629315, 0.0651228142271108, 0.22838472050995764], "top": "insufficient", "option_logits": null, "prompt_sha256": "8726e9f6e7ede98264291ff8ad0e45430c6ce3426738746ae7ec330c5225073c", "input_tokens": 192, "server_ms": 34.45}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92248393781483, "probs": [0.9439935183065499, 0.0409858846475417, 0.0150205970459084], "top": "B", "option_logits": null, "prompt_sha256": "bbdddbef606c62e9ac9a597096a51234b94035a57674ce04db983e275d076a86", "input_tokens": 203, "server_ms": 34.2}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16638591140509, "probs": [0.06642565405281799, 0.9261833743555155, 0.0073909715916665505], "top": "A", "option_logits": null, "prompt_sha256": "f9062a341f3150d9232fc2091c0f4a720c9f8a7a9ebf2a3b53f7945cc81b561f", "input_tokens": 203, "server_ms": 34.41}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.885753041133285, "probs": [0.9772298659128125, 0.012098405641787949, 0.010671728445399525], "top": "A", "option_logits": null, "prompt_sha256": "3efd8eb9f9fa0e03f2533097d8073cf8265b904bdaf5e12d7e71cbcda034ebae", "input_tokens": 193, "server_ms": 34.12}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0017250534147, "probs": [0.09931756389846377, 0.8926071429310916, 0.008075293170444685], "top": "insufficient", "option_logits": null, "prompt_sha256": "931bb93c879b95601b770f5bcc531d57aca3c51f1e201a92f41a4af0d559e6c5", "input_tokens": 192, "server_ms": 34.34}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.6703299805522, "probs": [0.06639387877077424, 0.007865755914174356, 0.9257403653150514], "top": "A", "option_logits": null, "prompt_sha256": "5d4a9ae116daa6161534b5519583cd012b52060548dce72ac7d0f781876c0ce5", "input_tokens": 208, "server_ms": 33.92}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03003516420722, "probs": [0.025425185481267775, 0.9673249441344611, 0.007249870384270957], "top": "B", "option_logits": null, "prompt_sha256": "6a579e3ca68fa3d28ee162c2bb9061af2372a8a728baca705895776608d58850", "input_tokens": 206, "server_ms": 34.33}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.752881947904825, "probs": [0.900473990544217, 0.08837786536850668, 0.01114814408727632], "top": "B", "option_logits": null, "prompt_sha256": "98330242bf9b4853692144bd273028a1065e63fecbd536b99f72041a124cbdde", "input_tokens": 200, "server_ms": 34.09}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.267087027430534, "probs": [0.1414487087141659, 0.4960584710750265, 0.3624928202108077], "top": "A", "option_logits": null, "prompt_sha256": "86f3509930c216c6bc17217666f92763bc1fa437854da9186a9a14e952bf3b17", "input_tokens": 193, "server_ms": 34.53}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.5348591003567, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 33.77}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.165816873312, "probs": [0.05584400041528186, 0.9398915591084774, 0.004264440476240678], "top": "A", "option_logits": null, "prompt_sha256": "104d6ac93e38d37c2cbd2fe1d413e52ca92dde1258cac35fc220528b73196b21", "input_tokens": 209, "server_ms": 34.46}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00524498708546, "probs": [0.04937458683512937, 0.008523069447562875, 0.9421023437173077], "top": "A", "option_logits": null, "prompt_sha256": "d217d38b7e6213bb691099e21c66a3df1e10fe6cde75e36ed398dd45fd11b8e9", "input_tokens": 194, "server_ms": 34.24}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43866994045675, "probs": [0.38826805862544134, 0.5657334002422568, 0.04599854113230185], "top": "insufficient", "option_logits": null, "prompt_sha256": "47a14c3c4a4be0db70f00c92b4b884aa58fae59d2d69bfaa15a6dd001c74977e", "input_tokens": 192, "server_ms": 34.69}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.819982808083296, "probs": [0.05525329298111996, 0.014797133073951734, 0.9299495739449284], "top": "contradicted", "option_logits": null, "prompt_sha256": "bb827fab3f7af6904a86bfc213ce42d7a2cf4557d68a9ad04f7b2711c1308a09", "input_tokens": 216, "server_ms": 33.94}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10033604875207, "probs": [0.8170038444540815, 0.05859532327955457, 0.12440083226636395], "top": "supported", "option_logits": null, "prompt_sha256": "17fe8aeab450890957725a1dad43750752505b560e3976c468dc3f41172f8a67", "input_tokens": 214, "server_ms": 34.27}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73769207671285, "probs": [0.9735218387884688, 0.00998474638199737, 0.01649341482953388], "top": "supported", "option_logits": null, "prompt_sha256": "238376c4cc5fbc7cd76f9ad9cb9e509ce751a7fd9c3b9a11286be50239d869ec", "input_tokens": 209, "server_ms": 33.99}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0814548805356, "probs": [0.9553471931059949, 0.01618550073428256, 0.02846730615972255], "top": "insufficient", "option_logits": null, "prompt_sha256": "ce06953b256550886167647265795ea3869e5ff8f5316771e55ce1518c69ddd5", "input_tokens": 207, "server_ms": 34.33}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.650230875238776, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 33.96}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13780711404979, "probs": [0.4209957860789075, 0.3076410529313528, 0.27136316098973967], "top": "permitted", "option_logits": null, "prompt_sha256": "d51944708b071233f3970b04f6c6336f8e70e3809df2ba49e0aabdd866ad8bdc", "input_tokens": 229, "server_ms": 34.32}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83741297572851, "probs": [0.05387521829671438, 0.906755673529099, 0.03936910817418649], "top": "permitted", "option_logits": null, "prompt_sha256": "8bff8d1de44bca4e15fd82d70b8f339fee2bd109ec9a92c9747b89a54e021644", "input_tokens": 222, "server_ms": 34.03}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90585307776928, "probs": [0.9459403411775168, 0.04372962453450628, 0.010330034287976873], "top": "insufficient", "option_logits": null, "prompt_sha256": "00c4b02298270b667458c96645eee6cce2724fb4365acaf92619d1772f6df80f", "input_tokens": 221, "server_ms": 34.12}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67480102740228, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.94}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91367292776704, "probs": [0.9863441380986024, 0.010771259663552918, 0.00288460223784459], "top": "A", "option_logits": null, "prompt_sha256": "ff213732225a439e059b46a57708fb2a00c963c13243e933f189700505c04cf5", "input_tokens": 210, "server_ms": 34.15}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.804131930693984, "probs": [0.14639175402671709, 0.7964845624457851, 0.05712368352749774], "top": "A", "option_logits": null, "prompt_sha256": "336f079f69772f717f7a6c3f445805134d5e53ef7b61fb0ad8ebcf52b418a553", "input_tokens": 209, "server_ms": 34.08}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79264197126031, "probs": [0.9283260069346223, 0.02597998847580144, 0.045694004589576176], "top": "insufficient", "option_logits": null, "prompt_sha256": "37ba551c43c8c232aa20632adb709ac0b29072c10a739d0094e4f45e5095146d", "input_tokens": 211, "server_ms": 34.09}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.926223896443844, "probs": [0.03832591832238971, 0.9398833253029221, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "f87a6aa4ea4be148073907bfbd331ed2bc23ab8a4dd102101ddfab9df68affa9", "input_tokens": 226, "server_ms": 34.06}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1317861136049, "probs": [0.15261824181393646, 0.7798668927662274, 0.06751486541983612], "top": "supported", "option_logits": null, "prompt_sha256": "414bff4be61247980ece53db3773f05e7b7026c988db7d1528e992a308174423", "input_tokens": 227, "server_ms": 34.37}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08279482088983, "probs": [0.02224167487575335, 0.959332479722117, 0.01842584540212969], "top": "supported", "option_logits": null, "prompt_sha256": "6f4011c71525c9b091a8816574bd328fde25b6ba7ab7b96ded01b3a730a3d931", "input_tokens": 202, "server_ms": 34.28}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3151779640466, "probs": [0.05252409837620808, 0.9412533141801623, 0.006222587443629545], "top": "insufficient", "option_logits": null, "prompt_sha256": "5c9d452b9464173f19e40416da519c21dd6ab53b7cde7ba01038e5e77b34ea1b", "input_tokens": 202, "server_ms": 34.5}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.006454076617956, "probs": [0.08806019173320215, 0.12050723322647901, 0.7914325750403188], "top": "prohibited", "option_logits": null, "prompt_sha256": "a8173efc87a69fb87bd0cca2d2a6bd8745dc3278ebad157ac546bb3f41987882", "input_tokens": 233, "server_ms": 34.16}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45924006216228, "probs": [0.5880203043739818, 0.24430850840437723, 0.16767118722164093], "top": "permitted", "option_logits": null, "prompt_sha256": "8aecf205f28a5179380e1ecfd6e459d887b2402409823f5046b6caa992897f9a", "input_tokens": 231, "server_ms": 34.61}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66213108226657, "probs": [0.8133507124046239, 0.05478605406972841, 0.13186323352564766], "top": "permitted", "option_logits": null, "prompt_sha256": "bcca8c96798eb11f3f4644bd6b48d9956d0ef797acb354a9d313e72d7decc3e6", "input_tokens": 216, "server_ms": 33.94}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21568710170686, "probs": [0.15147579954487175, 0.20728921703153022, 0.6412349834235981], "top": "insufficient", "option_logits": null, "prompt_sha256": "9109d62aa99372ff918463de6ec1fd8982d3a2a6d2840a8c9977e79bd2ab935b", "input_tokens": 223, "server_ms": 34.46}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76997194997966, "probs": [0.9535382502516264, 0.03221192217085971, 0.014249827577513809], "top": "A", "option_logits": null, "prompt_sha256": "a10dad7ab11ae42542c0d0b776d3f586dc16a28cb00984bccb5c89dc77075c2f", "input_tokens": 222, "server_ms": 33.99}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23562811501324, "probs": [0.04901004980173283, 0.9351466866030752, 0.015843263595192024], "top": "B", "option_logits": null, "prompt_sha256": "76d3d59ab8383e39c3b6c2101903ac065a99bf3d49d00c16e61ce375f0f7bc7c", "input_tokens": 222, "server_ms": 34.44}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11576503701508, "probs": [0.8768319468998427, 0.09162951349252214, 0.03153853960763513], "top": "B", "option_logits": null, "prompt_sha256": "83a38a50ea29d93de3d3e3463c307552985b65ef7fbecfb33f19f0c8b19cbe78", "input_tokens": 215, "server_ms": 34.24}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.253167014569044, "probs": [0.04595900357971523, 0.02033122661038582, 0.933709769809899], "top": "insufficient", "option_logits": null, "prompt_sha256": "5483a1f853a7dd5c96bd7c5c666a4e199aff64cda6caf94dcccb07b14eae9bfc", "input_tokens": 201, "server_ms": 34.41}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10237587802112, "probs": [0.04052399739483789, 0.026120737232413562, 0.9333552653727485], "top": "contradicted", "option_logits": null, "prompt_sha256": "dd4cc5375267c593d2a03b475e5c365a6578823acd2a704447d9b76168008824", "input_tokens": 223, "server_ms": 34.18}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25595795549452, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "supported", "option_logits": null, "prompt_sha256": "d4d2d98ff82872fd16a73e2e6edb11f4ecab9360d684a9727d45783d301f374f", "input_tokens": 219, "server_ms": 34.45}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90641303546727, "probs": [0.026724635554949777, 0.9549340005022864, 0.018341363942763936], "top": "supported", "option_logits": null, "prompt_sha256": "296f774f1cfc7e7122448c98bb6e3a52dc18135394acb94f5473c12727592101", "input_tokens": 220, "server_ms": 34.08}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10488602519035, "probs": [0.14195326239124903, 0.8223426360308026, 0.03570410157794831], "top": "insufficient", "option_logits": null, "prompt_sha256": "9b5808bbbac1566fd1810d826dd09771c70684f8e1c3c1505e9a93a728995747", "input_tokens": 218, "server_ms": 34.21}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00456488877535, "probs": [0.0674490002480741, 0.883262901735513, 0.04928809801641285], "top": "prohibited", "option_logits": null, "prompt_sha256": "dfbe932d725d8fa600a07771813057adf2f6237700a213c275d2418103feffd5", "input_tokens": 224, "server_ms": 34.07}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31611790135503, "probs": [0.17474597379856985, 0.4768198017317031, 0.34843422446972716], "top": "prohibited", "option_logits": null, "prompt_sha256": "faf8ebe7a328f98f41892d79a870c982c804cd410a13ccd121f1e747020314b3", "input_tokens": 221, "server_ms": 34.41}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9088239967823, "probs": [0.049073786622505555, 0.0715038981480174, 0.879422315229477], "top": "permitted", "option_logits": null, "prompt_sha256": "cd350bbed40603bfdec8aee19ed8f4aead08aae6c3e75e5950990320c9a1602f", "input_tokens": 221, "server_ms": 34.02}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12463588453829, "probs": [0.039087581768329945, 0.9002715143788037, 0.06064090385286648], "top": "insufficient", "option_logits": null, "prompt_sha256": "32a2ba65d112e44959b5167505e017413199fda0540cd2a995538aa795dabd11", "input_tokens": 220, "server_ms": 34.25}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82913212105632, "probs": [0.019768802923085988, 0.013567511661018838, 0.9666636854158951], "top": "A", "option_logits": null, "prompt_sha256": "6c3a458921faa1b3dc850b93cbb1d6ed2a1453f181ed0aeb0df4c5ae9cccf625", "input_tokens": 227, "server_ms": 34.14}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.248068023473024, "probs": [0.02394587818769938, 0.9700312518040514, 0.006022870008249252], "top": "B", "option_logits": null, "prompt_sha256": "0d78a5511291c0e8891da715a82ec2ad438e94f5dddaee4ff0a75c528e00ad7e", "input_tokens": 230, "server_ms": 34.38}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77031095139682, "probs": [0.028667490838461873, 0.9620652951327192, 0.00926721402881876], "top": "B", "option_logits": null, "prompt_sha256": "f925b35d49f8ca5aac7190490beee3fe1f89e7ea2183f46b6ff8de5291cc99d6", "input_tokens": 223, "server_ms": 34.07}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54799091257155, "probs": [0.022241675019567426, 0.01842584578673998, 0.9593324791936926], "top": "insufficient", "option_logits": null, "prompt_sha256": "836107065b1093b68346fcce5e1adc9aba25fe8c243775524b77da14167cf552", "input_tokens": 210, "server_ms": 34.86}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07323595881462, "probs": [0.055084494859207855, 0.9271085820941714, 0.017806923046620855], "top": "contradicted", "option_logits": null, "prompt_sha256": "a88acfe95a51b8a25895a91b153a497406a07078525e2b97bc3c403e06a540b7", "input_tokens": 227, "server_ms": 34.33}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33637789264321, "probs": [0.05252720299692694, 0.8840676728179807, 0.0634051241850924], "top": "supported", "option_logits": null, "prompt_sha256": "60d8999e12533dd71716a6a7531ccfc7ec7f2073d709301bdd11b0ac9658b776", "input_tokens": 224, "server_ms": 34.72}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96046490222216, "probs": [0.10359906505840241, 0.12505351454311125, 0.7713474203984862], "top": "insufficient", "option_logits": null, "prompt_sha256": "673717072f8a9221f1adf01785a87c22cc50bf85f7bb720009c52de94dae71e4", "input_tokens": 220, "server_ms": 34.31}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29049791581929, "probs": [0.04112297491157286, 0.011726020323995183, 0.9471510047644319], "top": "insufficient", "option_logits": null, "prompt_sha256": "63e2228bb7d5b362f1e995b7d1a0c9d791b2ec99854b33a504865fb77a469bad", "input_tokens": 208, "server_ms": 34.76}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.511437967419624, "probs": [0.2873316476131822, 0.23803638895290566, 0.4746319634339121], "top": "permitted", "option_logits": null, "prompt_sha256": "bf284395337716d2923af2416dde0e555c17bde0e0c5ea382e5010db93daadea", "input_tokens": 227, "server_ms": 34.01}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21550712361932, "probs": [0.18941262861483832, 0.8018310660162423, 0.008756305368919429], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ba65298c9d27b7445ffc33c5502792b0a58a03c427a8f2b68f062202870179", "input_tokens": 228, "server_ms": 34.59}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.846852861344814, "probs": [0.045449474437266706, 0.031192392974493464, 0.9233581325882398], "top": "permitted", "option_logits": null, "prompt_sha256": "81df99f7f22c5619613f932264037a494407bd0c9c1cd772d88cdb13bd93f038", "input_tokens": 225, "server_ms": 34.14}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32125787064433, "probs": [0.06610715660723597, 0.9217425227678987, 0.012150320624865412], "top": "insufficient", "option_logits": null, "prompt_sha256": "720a483b782512dcd38b2b0081ceb423a67c83842981d937ef45d7a508910f4c", "input_tokens": 218, "server_ms": 34.59}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.777301924303174, "probs": [0.4315972806039035, 0.5547093438603172, 0.013693375535779267], "top": "B", "option_logits": null, "prompt_sha256": "4e84221098fdfcea2eadf56d4d09744125fba5bd9394ccda4463841127ab114b", "input_tokens": 221, "server_ms": 34.12}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86732309870422, "probs": [0.8253003742309912, 0.1256641025406766, 0.04903552322833224], "top": "A", "option_logits": null, "prompt_sha256": "bd5f90a9191f0f7973e6d324cf6a49506e2cd54e3cfc5de85edbfda099804d5f", "input_tokens": 227, "server_ms": 34.14}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.7648818064481, "probs": [0.18006428080502881, 0.8116115753175274, 0.008324143877443896], "top": "A", "option_logits": null, "prompt_sha256": "e536eabe724f1b5913aff9c542223b70d01472d2b42b51b49d37b52e912739a5", "input_tokens": 207, "server_ms": 34.01}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.302068900316954, "probs": [0.0642990095007749, 0.09368810379830601, 0.842012886700919], "top": "insufficient", "option_logits": null, "prompt_sha256": "d1260818c5ac8e709aa2420a7f131bcdacd6f95c3ee49deeed13973d0a110795", "input_tokens": 197, "server_ms": 34.54}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87990317121148, "probs": [0.9649135185430301, 0.019733010925224156, 0.015353470531745737], "top": "contradicted", "option_logits": null, "prompt_sha256": "d3371673a5e78756742dbd1370ab4b5bf6fb4c9e37f1bf9f6b2f8fc296fa2b3d", "input_tokens": 213, "server_ms": 34.09}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0458047837019, "probs": [0.025219845781526033, 0.015267532821545986, 0.959512621396928], "top": "supported", "option_logits": null, "prompt_sha256": "11f0f8368eb8d7e7794ac745fa45e0bc9dc4b1ac9ed01cd9acc0b7783d4aaf6b", "input_tokens": 212, "server_ms": 34.36}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96564398705959, "probs": [0.9604600229721326, 0.012660690572914277, 0.026879286454953178], "top": "supported", "option_logits": null, "prompt_sha256": "3af4ed7c8a0907629d9650ab957ccf4b7c6a7dda4e2a6dd2995d5bb1743e52e3", "input_tokens": 202, "server_ms": 34.23}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.509040208533406, "probs": [0.027209446310095767, 0.8576061514878427, 0.11518440220206158], "top": "contradicted", "option_logits": null, "prompt_sha256": "c0ec0e2875ae2fe16b5af804b0c765df404b9b4f5d893843240547ff3251f572", "input_tokens": 203, "server_ms": 34.71}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16721688397229, "probs": [0.8574932428756057, 0.12262617681966377, 0.01988058030473045], "top": "prohibited", "option_logits": null, "prompt_sha256": "5d48756022f5bf582cc4dcaecef3a33be36788da1e16cde33b69afb8e41d6d0c", "input_tokens": 218, "server_ms": 34.44}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.749753100797534, "probs": [0.036422612390539545, 0.9510408592905502, 0.012536528318910257], "top": "permitted", "option_logits": null, "prompt_sha256": "1e8882942aa8fcb63c49bda41019547d5c388d1eb722d8f3d2f6ff0fe2f6de83", "input_tokens": 221, "server_ms": 34.94}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10572491399944, "probs": [0.03036360301441411, 0.9570210386636204, 0.012615358321965421], "top": "permitted", "option_logits": null, "prompt_sha256": "58d1fda856fe23c325b5eb40b4f3316df39757b9358721f866eaee369d56311c", "input_tokens": 215, "server_ms": 34.41}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.203877000138164, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "7e3b871d1d0377bf269820f9b1f1d027e38c15d4fc3b1ad841733cfa7eaa1caa", "input_tokens": 220, "server_ms": 34.5}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81704308837652, "probs": [0.9856903364066905, 0.012203148876708984, 0.0021065147166005127], "top": "A", "option_logits": null, "prompt_sha256": "49ab3244841a47c40baa28769654a2caa6beec9c9be7b8902fa30d48d3df3ae9", "input_tokens": 206, "server_ms": 34.13}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78688400052488, "probs": [0.026052379812935716, 0.043034915318676176, 0.9309127048683882], "top": "B", "option_logits": null, "prompt_sha256": "3e92d9a3ec71acb771522ae129c59a1c3f3cf135471b1a9119034e8e60f57534", "input_tokens": 212, "server_ms": 34.96}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93523397669196, "probs": [0.9773173707852877, 0.018771280462410337, 0.003911348752301854], "top": "B", "option_logits": null, "prompt_sha256": "c8b102c19db70a1095b4a0e3f5b0fc428173a9dc8eeba42de46dbbde084b4a9e", "input_tokens": 203, "server_ms": 34.2}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.232727045193315, "probs": [0.03778914105307011, 0.035491168622697584, 0.9267196903242324], "top": "insufficient", "option_logits": null, "prompt_sha256": "4a91f5a0c83e6f4ce998b7f04fc311297f577c57be12b145cf649e0aa08f1703", "input_tokens": 191, "server_ms": 34.55}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03547501936555, "probs": [0.7336632938165211, 0.04359087300806054, 0.2227458331754184], "top": "contradicted", "option_logits": null, "prompt_sha256": "71d93a27e5ea2ee304e786d87ef894fb7755249d7fb8613e14d533e8fd133800", "input_tokens": 224, "server_ms": 34.35}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37331903539598, "probs": [0.06296298480307037, 0.877902829429632, 0.05913418576729761], "top": "supported", "option_logits": null, "prompt_sha256": "ffb22e022a23a832611c31cce83e5ded9325f32d02e4447965afe9987da2bde6", "input_tokens": 221, "server_ms": 34.58}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81800304912031, "probs": [0.04392538756600811, 0.9501750287751061, 0.005899583658885871], "top": "supported", "option_logits": null, "prompt_sha256": "6075f3ad5119128a82986b4f0679052bba33e8403d8fb22ec4c8f7ed10d93946", "input_tokens": 229, "server_ms": 34.16}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.041175877675414, "probs": [0.9704787445693202, 0.01201482490914338, 0.01750643052153659], "top": "insufficient", "option_logits": null, "prompt_sha256": "48b9296dd657bda90e5f3b2c63f69eebef357ae1a9850de04e5cfd329a5ceeba", "input_tokens": 226, "server_ms": 34.32}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87272313982248, "probs": [0.3304014836417537, 0.5811151968205441, 0.08848331953770218], "top": "prohibited", "option_logits": null, "prompt_sha256": "f2f72ef9b5a8872981b046c25a1b9e9511405cfacc72a16d99f362fea882a19c", "input_tokens": 226, "server_ms": 34.23}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62884088791907, "probs": [0.11071640116478274, 0.011569936207086172, 0.877713662628131], "top": "permitted", "option_logits": null, "prompt_sha256": "13277d62009fb53093658090b337dad57eca4375e0abe61be5ee35f2ec7e4157", "input_tokens": 215, "server_ms": 34.89}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76699195057154, "probs": [0.17087454466914134, 0.3005367666543428, 0.5285886886765159], "top": "permitted", "option_logits": null, "prompt_sha256": "cecc81469d7fbe04a5075ff2cb74c748f517953000b249e9f63c274d01b5b1d9", "input_tokens": 225, "server_ms": 34.0}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28812816552818, "probs": [0.5363636986202766, 0.44434397563891254, 0.019292325740810894], "top": "insufficient", "option_logits": null, "prompt_sha256": "072c7c6a806afdb07685f8b1f3c43852c2e6b32f1c5516b65df03793dc2bb53f", "input_tokens": 224, "server_ms": 34.65}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94786410778761, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.21}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.401799112558365, "probs": [0.657302153396938, 0.15527126967431085, 0.1874265769287511], "top": "B", "option_logits": null, "prompt_sha256": "3b58b4916fb300ada6ef30438bc6d52ac081154f25e9849d672afc231c6278ff", "input_tokens": 218, "server_ms": 34.7}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91078396514058, "probs": [0.019992665974153154, 0.06184594114611621, 0.9181613928797306], "top": "B", "option_logits": null, "prompt_sha256": "d02d406417fcec2c9d2cbd7690bc752f474c89ea8c64602b69bbcceffa6d243a", "input_tokens": 209, "server_ms": 34.26}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10687602870166, "probs": [0.08689224299716879, 0.08160829342180988, 0.8314994635810213], "top": "insufficient", "option_logits": null, "prompt_sha256": "af78451b7d18177d0037af51b7830934b233d57812e77dd145bc82ebf8f94400", "input_tokens": 207, "server_ms": 34.36}], "seconds": 5.4, "failures": 0}, "negative": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.19386691227555, "probs": [0.06566437538338366, 0.8598927385077062, 0.07444288610891006], "top": "insufficient", "option_logits": null, "prompt_sha256": "2111f6636d10660f740e0f032908924d0640e60a712060af19c6e8972d95e5c9", "input_tokens": 204, "server_ms": 34.46}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.41741902194917, "probs": [0.2945662267563794, 0.03956295740237862, 0.665870815841242], "top": "contradicted", "option_logits": null, "prompt_sha256": "ed5aa8d62ce8778e40b8c51f1d1fe90b1f392ab252247908adc4e84b2546ab08", "input_tokens": 202, "server_ms": 34.73}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.99597413651645, "probs": [0.9241406947470572, 0.051569175242271295, 0.024290130010671554], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6f696700120ee981b3e3e0dc237f67896db84867803273e124be96bfd72fecc", "input_tokens": 196, "server_ms": 34.19}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.57349099032581, "probs": [0.11215130215053613, 0.052825544241278395, 0.8350231536081855], "top": "contradicted", "option_logits": null, "prompt_sha256": "ef4b71a8ae0a1bc3711450f77606ddabd2be62c9c2b6587e1fcacbccfcb9cb00", "input_tokens": 196, "server_ms": 34.86}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.05037501640618, "probs": [0.15307015041158983, 0.014109646797217717, 0.8328202027911924], "top": "insufficient", "option_logits": null, "prompt_sha256": "215840810bc2483aa303cd79f5d4f92093a49aa9957dfa697f20989aa1390d55", "input_tokens": 209, "server_ms": 34.3}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 38.261789130046964, "probs": [0.03329916574926079, 0.8166100641505288, 0.1500907701002104], "top": "supported", "option_logits": null, "prompt_sha256": "fc0d6607f95e1b53fe2cfa889a2cf1bfed2a95e2da0424c85742050fd8b5cde7", "input_tokens": 206, "server_ms": 35.5}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.627642042934895, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "supported", "option_logits": null, "prompt_sha256": "f3bf4ddaa8f4178e0b5f59c09fffe0c07bfd4bd34d616bbf9577361257ca1afa", "input_tokens": 208, "server_ms": 34.97}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 38.190389052033424, "probs": [0.3541639151292647, 0.5160413123689802, 0.1297947725017551], "top": "contradicted", "option_logits": null, "prompt_sha256": "e5578ff7fa77b4e9f8593ede9b8e31ecccdca126f02444f3c94cdd0bb01ee6ca", "input_tokens": 200, "server_ms": 35.5}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.17468702234328, "probs": [0.017835936327419456, 0.8721495360638406, 0.11001452760873995], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b2ae5222808313b49d4b84a395fe3c5b48345949fa231e2c8fba100ed470a94", "input_tokens": 194, "server_ms": 34.52}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.35720785334706, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b42ed76cec9fa4a467694c0069d312883b296f2ee2385059235881dee283be", "input_tokens": 194, "server_ms": 34.64}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.043055053800344, "probs": [0.03186734083364, 0.0247947114952375, 0.9433379476711226], "top": "contradicted", "option_logits": null, "prompt_sha256": "d2c60f4c3e6615ad87fdb4cd474622ea0a7f3f3b3fd7df582b5539d02a9dbd1b", "input_tokens": 193, "server_ms": 34.36}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.754462799057364, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "contradicted", "option_logits": null, "prompt_sha256": "89f547d1f4dbfccb3c2e3443f18b4576344bd16efeaec8108d2febb0c913ec9e", "input_tokens": 192, "server_ms": 35.0}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.04576496966183, "probs": [0.03540454160362492, 0.924457767599498, 0.04013769079687701], "top": "contradicted", "option_logits": null, "prompt_sha256": "6beeb2f79fa6330dc68f2b769cf2779997d4b57c459bf40113bbc11a80353072", "input_tokens": 205, "server_ms": 34.39}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.526621017605066, "probs": [0.028539740573724572, 0.7452090392508762, 0.22625122017539928], "top": "insufficient", "option_logits": null, "prompt_sha256": "b094a32aab8f6f92e09b2405572effe6bec478d6d3b5985cdeb71dab794ed591", "input_tokens": 205, "server_ms": 34.76}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.16989699751139, "probs": [0.026796810283021767, 0.844600378585805, 0.12860281113117333], "top": "insufficient", "option_logits": null, "prompt_sha256": "4e199755434441474e22e920ad5f8587827e30212d91945271ceadbb26337008", "input_tokens": 199, "server_ms": 34.43}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.30755904689431, "probs": [0.1303439116251751, 0.013621023271142042, 0.8560350651036829], "top": "contradicted", "option_logits": null, "prompt_sha256": "d75aa0e009db26e5e38e3bfd553573eccf95a0526360da23bada050f23e908f8", "input_tokens": 194, "server_ms": 34.58}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.960794823244214, "probs": [0.1369403800499024, 0.018392355065480374, 0.8446672648846173], "top": "supported", "option_logits": null, "prompt_sha256": "9e994209df771b9f8829dd54c36907e41ed241a507cce230c7e3505302a9fa1e", "input_tokens": 198, "server_ms": 34.3}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.425258895382285, "probs": [0.7651520472632017, 0.10276696393112311, 0.13208098880567515], "top": "contradicted", "option_logits": null, "prompt_sha256": "45785396c292fe5cbe62d99a45714d49c993d6f5b5783769cec1795f7b84f35f", "input_tokens": 198, "server_ms": 34.66}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.00388502329588, "probs": [0.9249067812606918, 0.05495367443647756, 0.020139544302830666], "top": "contradicted", "option_logits": null, "prompt_sha256": "490eb806b0921b46aa816c18534ea40c0719e89eb76c34feb4e89d55c921bf36", "input_tokens": 194, "server_ms": 34.28}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 38.0784971639514, "probs": [0.0190993007707109, 0.7736996936098133, 0.2072010056194758], "top": "contradicted", "option_logits": null, "prompt_sha256": "7e29e327fc2aef321d690607a7c7347871c7eb3dd07ac6d45327aab094603d91", "input_tokens": 194, "server_ms": 35.4}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.01690398156643, "probs": [0.021206655711830842, 0.5200601388697108, 0.45873320541845836], "top": "insufficient", "option_logits": null, "prompt_sha256": "ffb27e8dfc1d8298a601b7f483c42104d68f6e8ebfea086896afd140756b8c6c", "input_tokens": 211, "server_ms": 34.25}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.134605925530195, "probs": [0.9097361205627902, 0.05076536678270011, 0.039498512654509606], "top": "supported", "option_logits": null, "prompt_sha256": "e7855c00f037441fcdc5ff3fefaa61ddaeb0e79e09ba6c0c6043c767ee6d98ca", "input_tokens": 211, "server_ms": 34.46}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.734122946858406, "probs": [0.9223434223370909, 0.04004589025644397, 0.03761068740646525], "top": "supported", "option_logits": null, "prompt_sha256": "1a0c7ecb563009263a7969e56c6e00fc34f99b8ad08b4db579ed9e2bc78f4c6f", "input_tokens": 201, "server_ms": 34.86}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.467269925400615, "probs": [0.1051237595843464, 0.007539451491827663, 0.8873367889238258], "top": "contradicted", "option_logits": null, "prompt_sha256": "610dae3d86ea66ae02b343c18c1478ef66be2619fe3a6c03ff05a434fac307d6", "input_tokens": 197, "server_ms": 34.75}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.91238397732377, "probs": [0.7409240241997488, 0.18635729477403776, 0.07271868102621346], "top": "insufficient", "option_logits": null, "prompt_sha256": "52c15cca48120ff357156a28827735982f0407b36b9806dc79a94193e52ed2a4", "input_tokens": 212, "server_ms": 34.21}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.35221899114549, "probs": [0.9070425995126343, 0.07852475778895956, 0.01443264269840612], "top": "prohibited", "option_logits": null, "prompt_sha256": "5df6c8de75514db23d6e53210ba8920e11a83a08e31ffd76867eb6d789db3f29", "input_tokens": 218, "server_ms": 34.59}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.93469404242933, "probs": [0.04968326826324694, 0.890344493113758, 0.05997223862299509], "top": "insufficient", "option_logits": null, "prompt_sha256": "3dc91e8f64358377c40956ef40870819732d6e66fc8ed44530ee8b7b1e32f1e6", "input_tokens": 203, "server_ms": 34.24}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.31481800787151, "probs": [0.09374299764974467, 0.009200493913293934, 0.8970565084369614], "top": "prohibited", "option_logits": null, "prompt_sha256": "866701419f4e54bef33f2151b0e4c0c830728dbfd94c6ba4a3fbd92b438151a2", "input_tokens": 209, "server_ms": 34.56}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.79520194418728, "probs": [0.20575303419639995, 0.025954061735184695, 0.7682929040684153], "top": "prohibited", "option_logits": null, "prompt_sha256": "6c14eaaad2ab386d458a6458fce2b79bcd81ecbfc3c66679c950d808dd0c2a3f", "input_tokens": 209, "server_ms": 34.09}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.03748504631221, "probs": [0.8315971903507323, 0.10489918128594154, 0.06350362836332618], "top": "permitted", "option_logits": null, "prompt_sha256": "d8b705bc82e40643b8dcd135b43f67a8e3671ecc2f28e81b48de6aa1cbb84792", "input_tokens": 214, "server_ms": 34.15}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.17875713482499, "probs": [0.258643657502053, 0.6628301324407543, 0.07852621005719278], "top": "insufficient", "option_logits": null, "prompt_sha256": "5000e3b05641e28823047872c8bd19f4604d76e0fa38c23bdad3d8e7be7bce61", "input_tokens": 209, "server_ms": 34.47}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.151257041841745, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "prohibited", "option_logits": null, "prompt_sha256": "97ddaad2b5074dab6f31a5a7a8ded12c3e51123aa5d819f19c006ba123fde089", "input_tokens": 209, "server_ms": 34.39}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.83242201805115, "probs": [0.09658208517581993, 0.868022296339126, 0.03539561848505407], "top": "insufficient", "option_logits": null, "prompt_sha256": "5783bf781c97c761780f8d8454e12c8895251116d4d5ebe264401d9d9dcae0a1", "input_tokens": 208, "server_ms": 34.02}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.75034286081791, "probs": [0.0651099549445377, 0.02705164492548852, 0.9078384001299739], "top": "prohibited", "option_logits": null, "prompt_sha256": "a92081d320caa09114cab155b336249985672a93e86328503ed4397aa61c3293", "input_tokens": 211, "server_ms": 34.96}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.769642028957605, "probs": [0.049132112383896635, 0.729412594713007, 0.22145529290309635], "top": "insufficient", "option_logits": null, "prompt_sha256": "2b0f13909e189564a5e5099e9094600d4804ae6ef7ef59e7435f85302977a0c0", "input_tokens": 202, "server_ms": 34.04}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.362348986789584, "probs": [0.1638594512396548, 0.04974899459992729, 0.7863915541604178], "top": "prohibited", "option_logits": null, "prompt_sha256": "a4ffa08e79c520845fb80c116412fe4c48304f5f003b514bbcbea0a7c61cfe7b", "input_tokens": 204, "server_ms": 34.61}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.162496941164136, "probs": [0.06162034871506134, 0.8591822512326693, 0.07919740005226927], "top": "insufficient", "option_logits": null, "prompt_sha256": "f51b6f4e075e67f76ff574d496ba2f9c19cc2ca38a2f56d989791df67251d344", "input_tokens": 214, "server_ms": 34.42}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.437258986756206, "probs": [0.7621344563262837, 0.23138990123980285, 0.006475642433913465], "top": "insufficient", "option_logits": null, "prompt_sha256": "baf1674ea5ff54f508700c815711c06e9a50808663840042c147d883c72021ed", "input_tokens": 214, "server_ms": 34.63}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.99792409315705, "probs": [0.14398191537109406, 0.021923360098142518, 0.8340947245307634], "top": "insufficient", "option_logits": null, "prompt_sha256": "8854be3865426f818cb7189838d63dc61b9b64da67d13087c8e6ccc2a708a8c0", "input_tokens": 213, "server_ms": 34.28}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.36395901069045, "probs": [0.2129289153471605, 0.545676239155193, 0.24139484549764645], "top": "permitted", "option_logits": null, "prompt_sha256": "913468ed99926586c9292a6dabcee9b083a7b1b1c719b6bad31fda7f7f045a2e", "input_tokens": 212, "server_ms": 34.55}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.88133414834738, "probs": [0.15943768499721103, 0.025848589203191098, 0.814713725799598], "top": "insufficient", "option_logits": null, "prompt_sha256": "acfb724b1ffcab3d0820c0a1f4cdb189396280c1faf4e622ce1f306cad1bf45e", "input_tokens": 216, "server_ms": 34.14}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.50668000429869, "probs": [0.9471510047644319, 0.04112297491157286, 0.011726020323995183], "top": "insufficient", "option_logits": null, "prompt_sha256": "7a9df7814cddba377022c022111d998db3d8a2ff48c109eb7f1aac0f5c151319", "input_tokens": 216, "server_ms": 34.79}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.73869208432734, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "ecd27e22648cba485966b925a1b12fd5a888331b66634b0f8919798224cf8072", "input_tokens": 208, "server_ms": 33.97}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.067716009914875, "probs": [0.2729138208779564, 0.3294318709588002, 0.3976543081632435], "top": "insufficient", "option_logits": null, "prompt_sha256": "86759c52a7cc961194391587ca2df8d3998c59ec236637e4acc083f151ad5242", "input_tokens": 215, "server_ms": 34.37}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.87910293228924, "probs": [0.5472741025611921, 0.331307576591957, 0.12141832084685092], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ee81091d38ca686274af8a9057ae962583e07c84d4a7c8913246f20a9adc153", "input_tokens": 209, "server_ms": 34.15}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.95019497536123, "probs": [0.10569064817366915, 0.056437743537509265, 0.8378716082888217], "top": "insufficient", "option_logits": null, "prompt_sha256": "d890e328d92ed0294fd0ca0f45891d715f1b53fae8d48848ee13c936721cc9f2", "input_tokens": 212, "server_ms": 34.21}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.081576185300946, "probs": [0.745864682612535, 0.0779433517872937, 0.17619196560017134], "top": "permitted", "option_logits": null, "prompt_sha256": "866975030f9845962316982b48126dfbaa42cd89b8d3719305c5c1cd2a7ce35c", "input_tokens": 209, "server_ms": 34.36}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.40141890011728, "probs": [0.30671712087126574, 0.5394587819774863, 0.153824097151248], "top": "permitted", "option_logits": null, "prompt_sha256": "33ed49b156b02a8251a71c89906619f22d3cff122755a441bc701809f452ec03", "input_tokens": 210, "server_ms": 34.44}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.115626968443394, "probs": [0.9411304198850982, 0.03837677130689544, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "26a5967a0d4f86a5e648fdf739a14ae8463a48c7dc0319e4b9026afcfc70ce49", "input_tokens": 200, "server_ms": 34.25}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.410999881103635, "probs": [0.2191677303480843, 0.5616645393038313, 0.2191677303480843], "top": "insufficient", "option_logits": null, "prompt_sha256": "5cb564ef00f9a81eb96ac41fe01780b4b9ffdcb60d7bc8217b9d89860c279ba3", "input_tokens": 201, "server_ms": 34.57}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.89622296951711, "probs": [0.8246471790493917, 0.16138164146161363, 0.013971179488994609], "top": "insufficient", "option_logits": null, "prompt_sha256": "e747a01e40c9029fd1004867b0db224405c02f421c64f74cd748920300cb2000", "input_tokens": 192, "server_ms": 34.05}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.28801803663373, "probs": [0.9136677967042812, 0.07428832300095413, 0.012043880294764588], "top": "B", "option_logits": null, "prompt_sha256": "3d5649d07c298093a047c4d77967454e627c1fd4792f41873c4640c036a163cd", "input_tokens": 192, "server_ms": 34.45}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.10491489619017, "probs": [0.807231387784515, 0.17909249095841714, 0.013676121257067941], "top": "insufficient", "option_logits": null, "prompt_sha256": "88ac81fbf23c9684447ca373863f9cf4c54ca0bd35eb3e960680db4187456d96", "input_tokens": 208, "server_ms": 34.28}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.088045850396156, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "insufficient", "option_logits": null, "prompt_sha256": "7263692449eeef173103012af0a80bd27db0a89dc35bf50d4fed73b80a29f045", "input_tokens": 211, "server_ms": 34.29}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.97937401011586, "probs": [0.046119873005046885, 0.9369780144263187, 0.016902112568634386], "top": "B", "option_logits": null, "prompt_sha256": "4c2c0840c1ad7e181d30f10eb62a4618c3195e936034bc7274fe3c8fc945d786", "input_tokens": 200, "server_ms": 34.22}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.426428869366646, "probs": [0.6742384798146065, 0.29826788550402233, 0.02749363468137107], "top": "B", "option_logits": null, "prompt_sha256": "2d0f22653ae25b1b9ad253802809222e0554c7bd42292a5a1637a04128e5c52f", "input_tokens": 190, "server_ms": 34.72}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.84621304273605, "probs": [0.8763745016908776, 0.09751141040687612, 0.026114087902246324], "top": "insufficient", "option_logits": null, "prompt_sha256": "676f3a162c1d11c22fd5f07e5b57e6b6e6d3a4aae515a7c2965aeac124b616dd", "input_tokens": 203, "server_ms": 34.06}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.43135882541537, "probs": [0.13968031972950484, 0.23073250176366764, 0.6295871785068274], "top": "insufficient", "option_logits": null, "prompt_sha256": "109526bdb944267ab212d1b8ba2f77a3a6c7c147016cf89ae5bc2e0e8e661adc", "input_tokens": 206, "server_ms": 34.68}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.0277538895607, "probs": [0.9673249441344611, 0.025425185481267775, 0.007249870384270957], "top": "insufficient", "option_logits": null, "prompt_sha256": "c4889b691603599fd8dca223a645f5e88b7248a364a9bef64c37a8a9e3ced5eb", "input_tokens": 197, "server_ms": 34.33}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.27584797888994, "probs": [0.49248819366063284, 0.16951261805505752, 0.3379991882843097], "top": "insufficient", "option_logits": null, "prompt_sha256": "5deb35c130c31229df54964339f9faccffa08f3ed2eb38730d86ac9ccd510aa3", "input_tokens": 192, "server_ms": 34.49}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.85580489784479, "probs": [0.1558415388693792, 0.330859461294187, 0.5132989998364338], "top": "insufficient", "option_logits": null, "prompt_sha256": "7c27c7c3361bd04c584a975218ab6b610dcab0f9bf6078cf3870b470410c383c", "input_tokens": 203, "server_ms": 35.07}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.956225918605924, "probs": [0.30165258046080173, 0.13344430167683133, 0.5649031178623669], "top": "B", "option_logits": null, "prompt_sha256": "27dba9d4c38e8b61907893c325f5851e4726ba89fa7ac9e5563a14abe89239b9", "input_tokens": 203, "server_ms": 35.02}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.970815155655146, "probs": [0.9480355945752582, 0.038658346366163006, 0.013306059058578745], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f3dd4d0a864d976d5c35a7b088b3aed4b55500043acc8701dd4d35729125db8", "input_tokens": 193, "server_ms": 34.15}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.622781936079264, "probs": [0.22389674812152066, 0.7374540645868362, 0.03864918729164314], "top": "A", "option_logits": null, "prompt_sha256": "164c8915823587dee72ef8800598282728e580a02992fe7a8d8fa557ead4e3b9", "input_tokens": 192, "server_ms": 34.8}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.76367201842368, "probs": [0.9379420530362722, 0.04616732447007806, 0.01589062249364978], "top": "insufficient", "option_logits": null, "prompt_sha256": "d7c03673e95db552913947e72213d55f0938d187ca0d4fd69182cbe141bfda65", "input_tokens": 208, "server_ms": 33.96}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.2469681315124, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "A", "option_logits": null, "prompt_sha256": "b54ee64662a21acf112153e17512f10436c1938df35ab9ad182be1ff0c47c3e0", "input_tokens": 206, "server_ms": 34.49}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 36.84534295462072, "probs": [0.13579878113602123, 0.8376256951826758, 0.02657552368130303], "top": "A", "option_logits": null, "prompt_sha256": "8aa1b7f158bf57460e6f9f057913e0d01d913f16998cc0a46327e8aa6c93e838", "input_tokens": 200, "server_ms": 34.15}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.2057871427387, "probs": [0.25821814633948337, 0.25821814633948337, 0.4835637073210332], "top": "A", "option_logits": null, "prompt_sha256": "b17c06e9ca0e91accd4cfcb4667c5c3a42687d3209e1ecd98910eb1343a18dc2", "input_tokens": 193, "server_ms": 34.45}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.65228181146085, "probs": [0.12660014777476133, 0.09251256875552054, 0.7808872834697181], "top": "insufficient", "option_logits": null, "prompt_sha256": "324bb2046ede92322b8175d6af2961ad208eba5f52712692465ef944de2193a4", "input_tokens": 206, "server_ms": 34.81}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.483670050278306, "probs": [0.9299397393693557, 0.06263931303365462, 0.007420947596989686], "top": "insufficient", "option_logits": null, "prompt_sha256": "86e7d75bb9b41e2d9de272239185c5cce5c19dc1aac6573a737c31d47ab0563f", "input_tokens": 209, "server_ms": 34.75}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.86780296266079, "probs": [0.37290349783252347, 0.01111172675798736, 0.6159847754094891], "top": "insufficient", "option_logits": null, "prompt_sha256": "a1a07677dc816ad01403f9cff2867d5bfdaff2c29d8a4c2293ce0662a24f33ae", "input_tokens": 194, "server_ms": 34.12}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.52175997942686, "probs": [0.475785940079552, 0.1976777825029813, 0.3265362774174668], "top": "insufficient", "option_logits": null, "prompt_sha256": "82b8973ae0b97456180d256ef08822a11d20365d8039148ed863e9bb75c9bda4", "input_tokens": 192, "server_ms": 34.77}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.26044809445739, "probs": [0.14585095569321507, 0.009226876843769601, 0.8449221674630154], "top": "insufficient", "option_logits": null, "prompt_sha256": "9a672222518ab31da9c3ab2072849ef03b511e8e9d23762f89d33a83feca6ec1", "input_tokens": 216, "server_ms": 34.52}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.25271695293486, "probs": [0.09629593898139954, 0.7633942252849403, 0.14030983573366013], "top": "contradicted", "option_logits": null, "prompt_sha256": "5d2348206184a53d2da714668c8004522fc660c04abc09a4675170278b6d0ce8", "input_tokens": 214, "server_ms": 34.51}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.86699294485152, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "497e689287006b51bd4f178ebc9023050c338fca8abf790eef2112975de7f243", "input_tokens": 209, "server_ms": 34.12}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.45582001283765, "probs": [0.9369780144263187, 0.046119873005046885, 0.016902112568634386], "top": "supported", "option_logits": null, "prompt_sha256": "984a11157ce42cd37ec6209d96b53bf0d293025d6b69161f1a358f9bcc9d57db", "input_tokens": 207, "server_ms": 34.68}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.40770905278623, "probs": [0.6452505747343799, 0.2680863307491953, 0.08666309451642484], "top": "permitted", "option_logits": null, "prompt_sha256": "ea5a71f7bf051b93ca522c5ff3caef6ffdfcb5011f24af51a404b10524b250cb", "input_tokens": 228, "server_ms": 34.61}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.5995310023427, "probs": [0.43020843462499875, 0.1903144997976119, 0.3794770655773894], "top": "permitted", "option_logits": null, "prompt_sha256": "b5ac43b527f17cc117db3df096bff98320c5a248422c7b513eedeba99e619731", "input_tokens": 229, "server_ms": 34.86}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.67571092955768, "probs": [0.056437743537509265, 0.8378716082888217, 0.10569064817366915], "top": "insufficient", "option_logits": null, "prompt_sha256": "be77461c7c49b38137b436f74bcc533cf9fdcc1fbbcabe3acbe8c8c37f2272bd", "input_tokens": 222, "server_ms": 33.92}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.48444886878133, "probs": [0.1110623891201585, 0.008481107191351438, 0.88045650368849], "top": "permitted", "option_logits": null, "prompt_sha256": "12c28c53461194e05d3490b99aecb157c1965b942f5bfee5ef8163543d2bb1d0", "input_tokens": 221, "server_ms": 34.72}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.10500616580248, "probs": [0.7598853370160624, 0.23070704972212444, 0.009407613261813141], "top": "insufficient", "option_logits": null, "prompt_sha256": "0b7bc0317c30767061abcd7c39f9b71942a3c49b230cf334d1e6db7873b4dc61", "input_tokens": 216, "server_ms": 34.36}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.281058030202985, "probs": [0.9718671022270775, 0.019875215614779472, 0.008257682158143088], "top": "B", "option_logits": null, "prompt_sha256": "282f482d7bf4f65f4cc0f8dd6ca05d597f32f80783d49b010173ef79b130ffc5", "input_tokens": 210, "server_ms": 34.48}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.09457512013614, "probs": [0.08284327606718382, 0.0730741626697933, 0.8440825612630228], "top": "A", "option_logits": null, "prompt_sha256": "6bf8f97ede01d029938070b85d9e3aa545cbbbb84007fb406100f0a0a8ffa474", "input_tokens": 209, "server_ms": 34.38}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.55780099891126, "probs": [0.1753450388599366, 0.7903403209860906, 0.03431464015397285], "top": "A", "option_logits": null, "prompt_sha256": "33dc6c8523ece68e76db4eaffbb8a8470964368747977448381041503e3ad36c", "input_tokens": 211, "server_ms": 34.83}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.91759402863681, "probs": [0.03629282833642113, 0.01605512937357001, 0.9476520422900088], "top": "supported", "option_logits": null, "prompt_sha256": "6e4467088790119e033569f3066af7eedf2f72c934657e3a20082f685de045f2", "input_tokens": 226, "server_ms": 34.15}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.44473005644977, "probs": [0.044616056760944024, 0.6623675120954924, 0.2930164311435636], "top": "insufficient", "option_logits": null, "prompt_sha256": "1a718aab5e212abb4604d4e57ad3d7fb8cba756be66eba7d483eec463cb18c6b", "input_tokens": 227, "server_ms": 34.67}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.239558063447475, "probs": [0.04013769079687701, 0.03540454160362492, 0.924457767599498], "top": "contradicted", "option_logits": null, "prompt_sha256": "bafd2feb34ff693920168afa57cc5bc050eea256c82f2e9e071467805a7857f4", "input_tokens": 202, "server_ms": 34.52}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.95513487420976, "probs": [0.21057073282474073, 0.0031468025509083547, 0.7862824646243509], "top": "supported", "option_logits": null, "prompt_sha256": "c4c6394f10ee2df26d3dac75e02000ae56f28719ec24b10add7af6e77f5a4889", "input_tokens": 202, "server_ms": 35.28}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.758051021024585, "probs": [0.10502328955939726, 0.2865713245281423, 0.6084053859124604], "top": "permitted", "option_logits": null, "prompt_sha256": "37bd9a494e79ba1e91bbf2a68704609c66421b3ec11ab8a5bdd36908b17135b1", "input_tokens": 233, "server_ms": 34.05}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.23493800498545, "probs": [0.2680863307491953, 0.08666309451642484, 0.6452505747343799], "top": "insufficient", "option_logits": null, "prompt_sha256": "a115c9f1aaee5b00fa1fd3be52a599afa7112fdcf9eec8871ae98b3d6ef2183f", "input_tokens": 231, "server_ms": 34.43}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.848054034635425, "probs": [0.07184163899867897, 0.8298456430479688, 0.09831271795335216], "top": "prohibited", "option_logits": null, "prompt_sha256": "100e5f7a276a412d51e2ddf940777a1ff109ea7803a8291d638760c392064d80", "input_tokens": 216, "server_ms": 34.13}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.155676167458296, "probs": [0.7469915530638082, 0.14618461002062086, 0.106823836915571], "top": "prohibited", "option_logits": null, "prompt_sha256": "6486b8dc13e88f4d5e489f3b3b76c6bb5aeed63f19005132e10380f0181726aa", "input_tokens": 223, "server_ms": 34.45}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.87963401898742, "probs": [0.8638851317960057, 0.051328051461926705, 0.08478681674206763], "top": "B", "option_logits": null, "prompt_sha256": "941cd3d853797aa1a92c8261e9339af72809062c590b26ef2db8bdbbc5d68191", "input_tokens": 222, "server_ms": 34.16}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.095065927132964, "probs": [0.046253998446132885, 0.9397029314342114, 0.014043070119655722], "top": "A", "option_logits": null, "prompt_sha256": "f9c6ae0da8d46bec4df2b4102f3aebb45748add11af2d63a981d879fea6a717c", "input_tokens": 222, "server_ms": 34.29}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.75207193009555, "probs": [0.04067015270682889, 0.7760155022030047, 0.1833143450901663], "top": "insufficient", "option_logits": null, "prompt_sha256": "4df509501a3fa1e95f377f6b2cd7ff2737ebe52dd6e95472a1a35d0fc43e3fdf", "input_tokens": 215, "server_ms": 33.92}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.52150107175112, "probs": [0.05543534738188995, 0.9330136575005004, 0.01155099511760962], "top": "A", "option_logits": null, "prompt_sha256": "d68329bad204407dddf0266c2d55c897537804b87978786e2c29649197957a18", "input_tokens": 201, "server_ms": 34.68}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.070794962346554, "probs": [0.06878691906304032, 0.03042978117418096, 0.9007832997627787], "top": "supported", "option_logits": null, "prompt_sha256": "3fdb716c314c67c93413daa3cd5a38b13bec265f706d90a97c43e5a108104f13", "input_tokens": 223, "server_ms": 34.21}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.215736927464604, "probs": [0.06893683236028562, 0.08321304657583695, 0.8478501210638775], "top": "contradicted", "option_logits": null, "prompt_sha256": "d68195061969b189d71df192da0191d4d56877cf1d8ce0d4479bc8898c998c94", "input_tokens": 219, "server_ms": 34.35}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.12849598377943, "probs": [0.016115540131028547, 0.8390475816237557, 0.14483687824521582], "top": "insufficient", "option_logits": null, "prompt_sha256": "6edd6bca63afcb6fb6d263e3e248e24e556fde2b01d965155729342ef37e411e", "input_tokens": 220, "server_ms": 34.26}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.389439065009356, "probs": [0.2862501142494634, 0.02478133063598072, 0.6889685551145559], "top": "contradicted", "option_logits": null, "prompt_sha256": "d113705714c5c456d527c8466aeda96099d32a8069c13bf0f40b1ff3ed43ad11", "input_tokens": 218, "server_ms": 34.45}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.20997693017125, "probs": [0.1609098519153101, 0.7722359149508133, 0.06685423313387646], "top": "insufficient", "option_logits": null, "prompt_sha256": "19fd16742b6e54b9e2452e09592de10d5fc44c96cab1630e5f7651b1af77d898", "input_tokens": 224, "server_ms": 34.36}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.42909897118807, "probs": [0.3043910694191852, 0.39121786116162954, 0.3043910694191852], "top": "insufficient", "option_logits": null, "prompt_sha256": "ea512df6fc353426e37889a53cae2118b9e832157a4af067cb4a85489a5c83ff", "input_tokens": 221, "server_ms": 34.52}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.99270403012633, "probs": [0.08097077759111387, 0.04060828666999563, 0.8784209357388906], "top": "prohibited", "option_logits": null, "prompt_sha256": "71ad1455eaea177dccc5d9fc4078b19378e2ce1c1d27289d9120a1c445349487", "input_tokens": 221, "server_ms": 34.14}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.45562001131475, "probs": [0.8979988326296015, 0.06048782007179681, 0.041513347298601795], "top": "permitted", "option_logits": null, "prompt_sha256": "31939a3c71e268cd4751d57ccd6a07ff44f1ca1f81c7c2f7bf165ea50668fe0a", "input_tokens": 220, "server_ms": 34.55}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.89691284671426, "probs": [0.9821934763214742, 0.008345422472445314, 0.009461101206080565], "top": "B", "option_logits": null, "prompt_sha256": "13f4ee2ce11c253ccf784222a18f900b57830384e577f4adf0030d9893ed01a9", "input_tokens": 227, "server_ms": 34.04}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.23857691511512, "probs": [0.9584447268648851, 0.039082802533792566, 0.0024724706013222815], "top": "insufficient", "option_logits": null, "prompt_sha256": "c8a6bf0c918df98324f40d3c989bf2d3593c8114ebaba9510a0de7a63d90c43f", "input_tokens": 230, "server_ms": 34.38}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.863152869045734, "probs": [0.8152011530118691, 0.1808606650321593, 0.003938181955971634], "top": "insufficient", "option_logits": null, "prompt_sha256": "d512a04940165a8dc92bcf5e8a905fa84148aca8078cba93d34897ca73ebe5fc", "input_tokens": 223, "server_ms": 34.1}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.26520691998303, "probs": [0.9509567466006987, 0.028336479497072638, 0.020706773902228682], "top": "A", "option_logits": null, "prompt_sha256": "726c7f274cbd758a0855ced1b8e06e30b0888e289eeb459d087a868fb9610ef5", "input_tokens": 210, "server_ms": 34.52}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.839693086221814, "probs": [0.03787087428859695, 0.033405034339472804, 0.9287240913719302], "top": "supported", "option_logits": null, "prompt_sha256": "0032df826a7e49e7be6e1c8d5f96ece8822a858eeb6f5280c55a79fe2e628bbb", "input_tokens": 227, "server_ms": 34.1}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.23681811243296, "probs": [0.04458832792770586, 0.7990410839770169, 0.15637058809527726], "top": "insufficient", "option_logits": null, "prompt_sha256": "c29cf273a3d78467e10369969a81a42df358b37cb972ad44f78d455f8785b7e8", "input_tokens": 224, "server_ms": 34.46}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.80593194440007, "probs": [0.6079293570442296, 0.046423581791889615, 0.34564706116388083], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b9e9fd277a7252eb95c0dea4201d1d75394e8dd03ee09f576f6fc0355881f3", "input_tokens": 220, "server_ms": 34.09}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.465049885213375, "probs": [0.07421350543321283, 0.7101726109045401, 0.2156138836622472], "top": "contradicted", "option_logits": null, "prompt_sha256": "25fc9b80402930a472ddf5fbb9527f22c905747ffefc59e4865da0bcf422e5bb", "input_tokens": 208, "server_ms": 34.77}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.0442362036556, "probs": [0.3596689227288795, 0.5240624893078483, 0.11626858796327226], "top": "prohibited", "option_logits": null, "prompt_sha256": "1a9834724cc432424395c6e3e4989cae7698e00f85fedac3d9a8cd1a698cd265", "input_tokens": 227, "server_ms": 34.18}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.022263975813985, "probs": [0.17244943152985545, 0.1836151222987057, 0.6439354461714388], "top": "permitted", "option_logits": null, "prompt_sha256": "cc59abf9dfd3346910e8d93dc78fe4cd21d0c20962370b801e609b82712c7ea7", "input_tokens": 228, "server_ms": 34.3}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.97918402031064, "probs": [0.11048645425133781, 0.06688603847853027, 0.8226275072701319], "top": "insufficient", "option_logits": null, "prompt_sha256": "45e75124dce70ee83cbbd2df43d7c5f3045b0da83559f8b4772e8cb6fa82dbb0", "input_tokens": 225, "server_ms": 34.21}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.14300598949194, "probs": [0.07600853330506088, 0.8779775900702179, 0.046013876624721124], "top": "permitted", "option_logits": null, "prompt_sha256": "902f5bf4dd52878d854ff660af610beb5cd00553cb6cf9fa5c1e7a5ea010391b", "input_tokens": 218, "server_ms": 34.38}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.10998594760895, "probs": [0.07014840077352269, 0.16883849753435962, 0.7610131016921177], "top": "B", "option_logits": null, "prompt_sha256": "d4a4f3c22ce6befbff78058a364e4617bda379057c95b5bacf2410a162ea27f7", "input_tokens": 221, "server_ms": 34.22}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.102445028722286, "probs": [0.7195969840802272, 0.2637194956142592, 0.016683520305513583], "top": "insufficient", "option_logits": null, "prompt_sha256": "fc0371a2d78013d514b134511d38b37de6e71a39ad2fa9b95b0119945db7a316", "input_tokens": 227, "server_ms": 34.38}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.720062140375376, "probs": [0.6328238128438389, 0.029254643239392513, 0.33792154391676854], "top": "A", "option_logits": null, "prompt_sha256": "5521ebbd2304bdf6049132465a2b8c7b28c862f462f50f98d90efeb483601e64", "input_tokens": 207, "server_ms": 34.0}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.46623988263309, "probs": [0.22351946247571317, 0.12708523693891682, 0.64939530058537], "top": "A", "option_logits": null, "prompt_sha256": "c11ad01ccd57e2b3b849c07b89887124412bce4a8e5506adaeb954f23c598ac7", "input_tokens": 197, "server_ms": 34.8}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.69857094064355, "probs": [0.038658346366163006, 0.013306059058578745, 0.9480355945752582], "top": "supported", "option_logits": null, "prompt_sha256": "8d5e868c2065409c5dcedc10c1d875c102dfbb48d2433b234f0ad0456f1049bc", "input_tokens": 213, "server_ms": 34.06}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.242257967591286, "probs": [0.01953747345276732, 0.9553520377742281, 0.025110488773004678], "top": "contradicted", "option_logits": null, "prompt_sha256": "6ac7738979b2b31bef2f7e2f3d8984b0b2b559ee71fa4d2b728e28884e3183b5", "input_tokens": 212, "server_ms": 34.49}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.22202708013356, "probs": [0.05100292527189731, 0.035003776957469146, 0.9139932977706335], "top": "contradicted", "option_logits": null, "prompt_sha256": "66c4b4c8b3dc52b623b51acf50a373bf625ad5559da7c3bf90c47f56e7c2829b", "input_tokens": 202, "server_ms": 34.46}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.30594785884023, "probs": [0.09842495451008554, 0.01699017293024806, 0.8845848725596664], "top": "insufficient", "option_logits": null, "prompt_sha256": "7db376c4d8c96d2c69fda861429911890ec2d70b0f375e60f65c846f1693e18a", "input_tokens": 203, "server_ms": 34.5}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.63808084093034, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "03449c2d41ee64f00d22b4b6e619aa69ce6d623d00b4972f503f5ed336bba08e", "input_tokens": 218, "server_ms": 33.96}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.38370887003839, "probs": [0.9331701885712249, 0.04890646327936172, 0.017923348149413328], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ed1d2ff423bdb9e622336508299a3180255754a84d1847b9b5ba5d1d5623692", "input_tokens": 221, "server_ms": 34.65}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.685761064291, "probs": [0.05755147392994642, 0.0327217252711575, 0.9097268007988961], "top": "insufficient", "option_logits": null, "prompt_sha256": "fb0073f867c77a04f9287adbd88276517a2a85e5cf9dcfb04c81ff5ba768e03f", "input_tokens": 215, "server_ms": 33.97}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.47096983715892, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "insufficient", "option_logits": null, "prompt_sha256": "51c4ac799c31ceb752046516252cb8d59c3a3c1654c685c82476530a4d74a689", "input_tokens": 220, "server_ms": 34.8}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.83291212655604, "probs": [0.040967527214491276, 0.8323022347202245, 0.12673023806528433], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a5a9b08e7ca06d1f5808cdcd3270c79b064cd4fed16b4be278fc7d5f58a1d1f", "input_tokens": 206, "server_ms": 34.08}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.33674809336662, "probs": [0.048847636440916636, 0.07578275550521898, 0.8753696080538644], "top": "insufficient", "option_logits": null, "prompt_sha256": "555fd1e019506f048448ae23a6bac4fe7f3e9f2fe60181be02acfdf8225318e5", "input_tokens": 212, "server_ms": 34.49}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.185137160122395, "probs": [0.07319171247064139, 0.2264135372721482, 0.7003947502572104], "top": "insufficient", "option_logits": null, "prompt_sha256": "1fc713f5dfc86b0d90f75094b087d920fbfeb9830fb4eec74f78d2f1d67fbf66", "input_tokens": 203, "server_ms": 34.47}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.423179019242525, "probs": [0.9387496456243781, 0.055776151759141294, 0.005474202616480596], "top": "B", "option_logits": null, "prompt_sha256": "77e6633b56fd111f3cf38ff30d1c308c084b3de608efb9682cf02409dc31c9db", "input_tokens": 191, "server_ms": 34.6}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.90357296727598, "probs": [0.46903312718147117, 0.38856479455009935, 0.14240207826842943], "top": "supported", "option_logits": null, "prompt_sha256": "83104242d8900e7e51e6ca6f99d29588121dd7153c0fda9d487e3eafef91196e", "input_tokens": 224, "server_ms": 34.15}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.522150902077556, "probs": [0.7720397533404616, 0.11755438599864407, 0.11040586066089443], "top": "insufficient", "option_logits": null, "prompt_sha256": "66f0b52c1eee120f16069ed75fbd7bfc556b5a3dc09c27f809799b32ae74e959", "input_tokens": 221, "server_ms": 34.8}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.94720286875963, "probs": [0.014236196905258059, 0.6140385266223682, 0.3717252764723737], "top": "insufficient", "option_logits": null, "prompt_sha256": "fac76f29223ea6573fb84bb8cb70a27292765e7f4f90ebb19001e8a0b7bf0533", "input_tokens": 229, "server_ms": 34.19}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.23230701871216, "probs": [0.06291220787817095, 0.0030966605414949147, 0.9339911315803341], "top": "supported", "option_logits": null, "prompt_sha256": "89df7654ae643d961d5bd0a1cf472717b674fe14a8f5377a6918c1103303e14f", "input_tokens": 226, "server_ms": 34.46}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.92846396006644, "probs": [0.5216500126916775, 0.26161676918070537, 0.21673321812761712], "top": "prohibited", "option_logits": null, "prompt_sha256": "73a309dd6a1605e1d928e2a4947516f3ae54fda513d77cea9d4fdf892dbbebb5", "input_tokens": 226, "server_ms": 34.22}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.32068813405931, "probs": [0.843310768581531, 0.1455727910071266, 0.01111644041134243], "top": "prohibited", "option_logits": null, "prompt_sha256": "97510bceb98f70589de7351d3f810c2e4c175b03bbf6ddec55eca00ef9c199ee", "input_tokens": 215, "server_ms": 34.48}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.94048407487571, "probs": [0.42929779219364844, 0.486689574582582, 0.08401263322376958], "top": "permitted", "option_logits": null, "prompt_sha256": "5a1d1ca84275c476cd245cafcd0e1fd36598a6dae3c643b8ea9d60c5b57d6535", "input_tokens": 225, "server_ms": 34.18}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.29790891520679, "probs": [0.19560247361171754, 0.23611000151056785, 0.5682875248777147], "top": "permitted", "option_logits": null, "prompt_sha256": "fec6fd7a8c54ee7856b56b8cc85aa3ff4cb617820fd52f7faf204e2efb4a86ff", "input_tokens": 224, "server_ms": 34.59}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.3271678108722, "probs": [0.9323898466010938, 0.02450695094155103, 0.04310320245735524], "top": "B", "option_logits": null, "prompt_sha256": "fcc6cca56205c973fe5ee42de7bbca225406234b5346f63e55af7d8c5f115d9a", "input_tokens": 213, "server_ms": 34.58}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.224116967990994, "probs": [0.643833389199536, 0.3438005390976864, 0.012366071702777563], "top": "insufficient", "option_logits": null, "prompt_sha256": "176f3f577d5802518ba6bb8a36039f8c2c7fa4991aac2e236870e0189d0b31a6", "input_tokens": 218, "server_ms": 34.58}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.18743706122041, "probs": [0.020042268594912593, 0.21743089555343206, 0.7625268358516554], "top": "B", "option_logits": null, "prompt_sha256": "8494413c37184f76cf07859eef5fe11f7373b1acb0da7cbd005226a5bb00610d", "input_tokens": 209, "server_ms": 34.45}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 40.08654900826514, "probs": [0.33058297459294167, 0.050336110784327995, 0.6190809146227304], "top": "A", "option_logits": null, "prompt_sha256": "70f353d85ca31feb6cd6d5c421fe7a2e877a313657d44902068dc2f6ef451bf8", "input_tokens": 207, "server_ms": 37.31}], "seconds": 5.4, "failures": 0}, "multifield": {"rows": [{"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.74664808064699, "probs": [0.14720923592665844, 0.8527907640733415], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.74664808064699, "probs": [0.8214121479792841, 0.17115665028167817, 0.00743120173903778], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.74664808064699, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.74664808064699, "probs": [0.9305404571279347, 0.06673814075012655, 0.0027214021219388934], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.93118519894779, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.93118519894779, "probs": [0.8095965925134007, 0.07462670936879237, 0.11577669811780701], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.93118519894779, "probs": [0.9471470189565372, 0.052852981043462664], "top": "same"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.93118519894779, "probs": [0.9350414422025257, 0.05915293776423124, 0.005805620033243134], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.766634004190564, "probs": [0.22187782400103293, 0.778122175998967], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.766634004190564, "probs": [0.09874410354776683, 0.06776898998409085, 0.8334869064681424], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.766634004190564, "probs": [0.07967467169628852, 0.9203253283037116], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.766634004190564, "probs": [0.10841793826059654, 0.0843556864709907, 0.8072263752684127], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.014336006715894, "probs": [0.21123545993246395, 0.788764540067536], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.014336006715894, "probs": [0.11796301878836102, 0.8782943442297925, 0.0037426369818465326], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.014336006715894, "probs": [0.8527907640733415, 0.14720923592665844], "top": "same"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.014336006715894, "probs": [0.24350964936246988, 0.7532804299591004, 0.0032099206784296486], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.67681306414306, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.67681306414306, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.67681306414306, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.67681306414306, "probs": [0.9344875966425453, 0.06294564908924384, 0.002566754268210852], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.40572818182409, "probs": [0.7318060856600669, 0.2681939143399331], "top": "moved"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.40572818182409, "probs": [0.727805131100519, 0.25050783114702185, 0.02168703775245924], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.40572818182409, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.40572818182409, "probs": [0.8387192579214904, 0.15415437985672864, 0.007126362221781067], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.93817188590765, "probs": [0.10592235395880453, 0.8940776460411956], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.93817188590765, "probs": [0.928892622801621, 0.06661995929549502, 0.004487417902884053], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.93817188590765, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.93817188590765, "probs": [0.9588514552888884, 0.039099387288032916, 0.002049157423078645], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.50075894035399, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.50075894035399, "probs": [0.14648079183179633, 0.8485708719675852, 0.004948336200618522], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.50075894035399, "probs": [0.9248060506082927, 0.07519394939170732], "top": "same"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.50075894035399, "probs": [0.24334788832895243, 0.7527800186067107, 0.0038720930643368974], "top": "meet_wren"}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.63415399752557, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.63415399752557, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.63415399752557, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.63415399752557, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.644794987514615, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.644794987514615, "probs": [0.8089141677995819, 0.1910858322004182], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.644794987514615, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.644794987514615, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.80885700508952, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.80885700508952, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.80885700508952, "probs": [0.9439168356016766, 0.0560831643983235], "top": "same"}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.80885700508952, "probs": [0.9156019658234928, 0.08439803417650704], "top": "stay"}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.294302843511105, "probs": [0.3201997500236374, 0.6798002499763626], "top": "holds"}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.294302843511105, "probs": [0.12511350509772579, 0.8748864949022742], "top": "departure"}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.294302843511105, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.294302843511105, "probs": [0.13214384255458153, 0.8678561574454184], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.847167890518904, "probs": [0.25606104458540857, 0.7439389554145914], "top": "holds"}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.847167890518904, "probs": [0.08439803417650704, 0.9156019658234928], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.847167890518904, "probs": [0.11840617139815589, 0.8815938286018441], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.847167890518904, "probs": [0.10012611628383591, 0.8998738837161642], "top": "departure"}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.15972091071308, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.15972091071308, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.15972091071308, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.15972091071308, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.34255303628743, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.34255303628743, "probs": [0.8184242551251981, 0.18157574487480185], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.34255303628743, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.34255303628743, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.16431209817529, "probs": [0.12511350509772579, 0.8748864949022742], "top": "holds"}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.16431209817529, "probs": [0.9248060506082927, 0.07519394939170732], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.16431209817529, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.16431209817529, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.21320094540715, "probs": [0.7193152314525416, 0.2806847685474584], "top": "moved"}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.21320094540715, "probs": [0.08937422170979005, 0.91062577829021], "top": "outside"}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.21320094540715, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.21320094540715, "probs": [0.07519394939170732, 0.9248060506082927], "top": "outside"}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.52346617169678, "probs": [0.15526082656247686, 0.844739173437523], "top": "holds"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.52346617169678, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.52346617169678, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.52346617169678, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.119541093707085, "probs": [0.45308504361568214, 0.5469149563843178], "top": "holds"}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.119541093707085, "probs": [0.25606104458540857, 0.7439389554145914], "top": "outside"}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.119541093707085, "probs": [0.15526082656247686, 0.844739173437523], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.119541093707085, "probs": [0.43758935794935333, 0.5624106420506467], "top": "outside"}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.81925887800753, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.81925887800753, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.81925887800753, "probs": [0.9203253283037116, 0.07967467169628852], "top": "same"}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.81925887800753, "probs": [0.936892368084393, 0.06310763191560703], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.8634779099375, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.8634779099375, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.8634779099375, "probs": [0.9156019658234928, 0.08439803417650704], "top": "same"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.8634779099375, "probs": [0.91062577829021, 0.08937422170979005], "top": "stay"}], "seconds": 1.0, "failures": 0}}, "finished_utc": "2026-09-30T16:24:06Z"} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/shape.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/shape.json new file mode 100644 index 0000000..577b9a1 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/shape.json @@ -0,0 +1 @@ +{"label": "r1", "backend": "semif", "runs": 3, "per": 20, "max_questions": 10000, "events": [["start", 1790785446.6640766], ["short1:measure", 1790785446.7472284], ["crit21:measure", 1790785449.1073372], ["long1:measure", 1790785460.8315358], ["long16:measure", 1790785479.047047], ["shapes:done", 1790785492.1825504], ["end", 1790785492.1825526]], "shapes": {"short1": {"rows": 1, "runs": [{"e2e_ms": [36.492628045380116, 35.91494192369282, 36.392577923834324, 36.122343968600035, 36.65242996066809, 36.191864870488644, 36.36887692846358, 36.27404710277915, 36.34133702144027, 36.191144958138466, 36.61168017424643, 36.17547499015927, 36.51740890927613, 36.49050812236965, 36.453508073464036, 36.251805955544114, 36.415617913007736, 36.19445604272187, 36.61038004793227, 36.248215939849615], "server_ms": [34.88988080061972, 34.50657590292394, 34.77428900077939, 34.46928597986698, 34.92498095147312, 34.534256206825376, 34.721018048003316, 34.62172788567841, 34.653868060559034, 34.505436196923256, 34.96109088882804, 34.5119358971715, 34.779808949679136, 34.71215791068971, 34.827849827706814, 34.61959701962769, 34.75238895043731, 34.54940696246922, 34.90631002932787, 34.50681618414819], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.35510697495192, "median_server": 34.68301298562437}, {"e2e_ms": [36.80864302441478, 36.47763794288039, 36.265225848183036, 36.32116713561118, 36.68301086872816, 35.96121305599809, 36.56349005177617, 36.546479910612106, 36.738761933520436, 36.32663702592254, 36.46715893410146, 36.325596971437335, 36.718221846967936, 36.52734914794564, 36.61821107380092, 36.28319688141346, 36.5538289770484, 36.403107922524214, 36.575459875166416, 36.11040487885475], "server_ms": [35.01950204372406, 34.81122897937894, 34.577697049826384, 34.66393705457449, 34.998820861801505, 34.31046404875815, 34.91510101594031, 34.765728982165456, 34.95729109272361, 34.69329793006182, 34.7655089572072, 34.643847960978746, 35.04138300195336, 34.717597998678684, 34.95334112085402, 34.62773794308305, 34.87422992475331, 34.75033910945058, 34.9161010235548, 34.47917499579489], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.50249354541302, "median_server": 34.76561896968633}, {"e2e_ms": [36.64528112858534, 36.28872614353895, 36.52584902010858, 36.39020700939, 36.50785912759602, 36.331726936623454, 36.5854490082711, 36.25095612369478, 36.73979081213474, 36.19861486367881, 36.59448120743036, 36.30744596011937, 36.452338099479675, 36.248445976525545, 36.693501053377986, 36.366238025948405, 36.45410807803273, 36.34117706678808, 36.488879006356, 36.31637594662607], "server_ms": [34.923350904136896, 34.64132687076926, 34.85312988050282, 34.742878982797265, 34.847409930080175, 34.6749781165272, 34.90375098772347, 34.550115931779146, 35.10982193984091, 34.5412059687078, 34.91210099309683, 34.64501700364053, 34.79888988658786, 34.573056967929006, 35.037842113524675, 34.68787902966142, 34.788229037076235, 34.73155898973346, 34.81566999107599, 34.67295691370964], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 181, "median_e2e": 36.421272554434836, "median_server": 34.76555400993675}], "tokens": 181, "median_e2e_ms": 36.4, "run_medians_e2e_ms": [36.4, 36.5, 36.4], "median_server_ms": 34.7, "non_200": 0}, "crit21": {"rows": 21, "runs": [{"e2e_ms": [82.19924313016236, 82.26463384926319, 82.67926913686097, 82.31112500652671, 82.31900492683053, 82.18895294703543, 82.24676409736276, 82.27341412566602, 82.31428498402238, 82.26709393784404, 82.21456315368414, 82.21651311032474, 82.32509507797658, 82.36855501309037, 82.32789393514395, 82.37709500826895, 82.18358410522342, 82.2441540658474, 82.23315398208797, 82.43833598680794], "server_ms": [78.7571941036731, 78.84017494507134, 79.26224893890321, 78.84746417403221, 78.87630490586162, 78.80562404170632, 78.85970501229167, 78.84615496732295, 78.91316595487297, 78.85352615267038, 78.83592485450208, 78.81302409805357, 78.9219350554049, 78.9923460688442, 78.94217502325773, 78.95944593474269, 78.78199405968189, 78.83897493593395, 78.83665501140058, 79.02529696002603], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 82.27025403175503, "median_server": 78.8504951633513}, {"e2e_ms": [82.1714629419148, 83.81023118272424, 82.38251507282257, 82.209262996912, 82.37030496820807, 82.40471594035625, 82.46810710988939, 82.2386140935123, 82.08175213076174, 82.35557493753731, 82.64597784727812, 82.16793299652636, 82.54825696349144, 82.28606404736638, 82.14497310109437, 82.21001294441521, 82.04889204353094, 82.15811289846897, 82.52505701966584, 82.40395598113537], "server_ms": [78.75512400642037, 80.32300183549523, 78.95342586562037, 78.82848405279219, 78.97318596951663, 79.05407599173486, 79.07085795886815, 78.81944486871362, 78.65468296222389, 78.99721711874008, 79.18951893225312, 78.75559292733669, 79.13928711786866, 78.91080598346889, 78.78314494155347, 78.82921490818262, 78.71557399630547, 78.72587302699685, 79.11896798759699, 78.94542603753507], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 82.32081949245185, "median_server": 78.92811601050198}, {"e2e_ms": [82.26742409169674, 82.39711495116353, 82.39720482379198, 82.56797702051699, 82.46130612678826, 82.3487548623234, 82.51670701429248, 82.31301512569189, 82.43447612039745, 82.36613497138023, 82.50988693907857, 82.54854800179601, 82.32063404284418, 82.3426938150078, 82.21565396524966, 84.21626593917608, 85.2127680554986, 85.17735684290528, 85.40427987463772, 85.24232916533947], "server_ms": [78.87378404848278, 79.03088605962694, 78.96392582915723, 79.10960796289146, 79.06630705110729, 78.93338589929044, 79.16128914803267, 78.95115693099797, 78.98716698400676, 78.96682596765459, 79.14093788713217, 79.15906794369221, 78.95180606283247, 78.9158851839602, 78.86044518090785, 78.82571499794722, 78.92149593681097, 78.90225504525006, 79.07458790577948, 79.1124370880425], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 82.44789112359285, "median_server": 78.96537589840591}], "tokens": null, "median_e2e_ms": 82.3, "run_medians_e2e_ms": [82.3, 82.3, 82.4], "median_server_ms": 78.9, "non_200": 0}, "long1": {"rows": 1, "runs": [{"e2e_ms": [197.47225288301706, 195.1804340351373, 188.85124707594514, 196.68696308508515, 190.03581209108233, 190.34004514105618, 190.2435428928584, 192.66265397891402, 188.21338890120387, 190.4432161245495, 195.535350125283, 198.2669320423156, 187.93371599167585, 194.00397106073797, 190.8497919794172, 191.85693399049342, 189.95952978730202, 200.29758801683784, 187.38102912902832, 197.73794617503881], "server_ms": [193.9458700362593, 191.75465195439756, 185.2949031163007, 193.2638199068606, 186.46060698665679, 186.82555085979402, 186.7539999075234, 189.14902000688016, 184.90071897394955, 187.02483386732638, 192.0915769878775, 194.88129997625947, 184.664994943887, 190.62001793645322, 187.48688884079456, 188.4525918867439, 186.5799780935049, 196.95607712492347, 184.04817790724337, 194.43143485113978], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 191.3533629849553, "median_server": 187.96974036376923}, {"e2e_ms": [190.931182121858, 190.15034288167953, 188.99440811946988, 197.775135980919, 187.34193802811205, 193.60240502282977, 190.44136605225503, 188.736654818058, 189.89336886443198, 195.66541002131999, 190.3315749950707, 195.24762593209743, 191.36429717764258, 194.6493680588901, 192.84297595731914, 191.69012201018631, 191.5578399784863, 206.996199907735, 184.70385600812733, 195.2032649423927], "server_ms": [187.48337007127702, 186.78028089925647, 185.72497693821788, 194.37124393880367, 183.92007611691952, 190.22568291984499, 187.10132502019405, 185.48704497516155, 186.55297788791358, 192.31892889365554, 186.9383230805397, 190.86261186748743, 186.89077207818627, 191.32745685055852, 189.48287493549287, 188.41821211390197, 188.22208885103464, 203.55334784835577, 181.33672489784658, 191.92242505960166], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 191.46106857806444, "median_server": 187.85272946115583}, {"e2e_ms": [194.73237893544137, 188.01285605877638, 193.99361009709537, 201.55627303756773, 187.17385502532125, 193.32327181473374, 191.34392705745995, 190.62913791276515, 191.70603202655911, 193.00870806910098, 190.02997200004756, 191.6908819694072, 192.70169408991933, 191.43051793798804, 189.5072851330042, 195.46239892952144, 189.5948159508407, 188.611552817747, 190.47313695773482, 191.3886379916221], "server_ms": [191.22551591135561, 184.68924495391548, 190.46839699149132, 197.9968089144677, 183.68152203038335, 189.8862300440669, 187.87933490239084, 187.12536501698196, 188.24358889833093, 189.54221508465707, 186.5190181415528, 188.27936006709933, 189.08247887156904, 187.75670300237834, 185.9166701324284, 191.83268281631172, 185.90618995949626, 185.01994921825826, 186.89140281639993, 187.8314740024507], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": 4019, "median_e2e": 191.40957796480507, "median_server": 187.85540445242077}], "tokens": 4019, "median_e2e_ms": 191.4, "run_medians_e2e_ms": [191.4, 191.5, 191.4], "median_server_ms": 187.9, "non_200": 0}, "long16": {"rows": 16, "runs": [{"e2e_ms": [227.92203701101243, 217.88762393407524, 219.8450278956443, 218.7842749990523, 215.56856599636376, 221.3859569747001, 213.5752709582448, 215.9328400157392, 216.85513202100992, 217.18715480528772, 225.54732905700803, 226.82262398302555, 211.20558213442564, 216.16054303012788, 216.47571679204702, 221.10197390429676, 214.04912695288658, 221.3276168331504, 214.05204804614186, 216.0082207992673], "server_ms": [222.1687869168818, 211.6728180553764, 213.44181918539107, 213.06140488013625, 209.8544251639396, 215.74249817058444, 207.85366115160286, 210.3161308914423, 211.380944121629, 211.51264617219567, 219.75482790730894, 221.03878296911716, 205.4936420172453, 210.36703209392726, 210.75089601799846, 215.362983988598, 208.403647178784, 215.62699601054192, 208.40541715733707, 210.25738096795976], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 217.02114341314882, "median_server": 211.44679514691234}, {"e2e_ms": [215.99016105756164, 216.5028969757259, 230.24138598702848, 228.791288100183, 211.57971606589854, 217.7568629849702, 214.5830038934946, 218.83109607733786, 217.14030602015555, 218.56013196520507, 214.70355498604476, 217.29583712294698, 219.3624428473413, 216.89611207693815, 237.1262509841472, 211.05049992911518, 215.92950983904302, 215.6399260275066, 216.8107710313052, 219.01213796809316], "server_ms": [210.29799105599523, 210.8859778381884, 224.51744601130486, 223.10825902968645, 205.7791450060904, 212.03265199437737, 208.91411392949522, 212.9904341418296, 211.437864927575, 212.95410301536322, 208.9606449007988, 211.49171609431505, 213.6256319936365, 211.11501078121364, 231.42162105068564, 205.27258911170065, 210.23227996192873, 209.99080687761307, 211.1094209831208, 213.23429699987173], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 217.01820904854685, "median_server": 211.27643785439432}, {"e2e_ms": [216.28470392897725, 214.82547605410218, 215.35645285621285, 216.44591679796576, 215.16175102442503, 222.991987131536, 214.8704279679805, 217.77986199595034, 217.83614391461015, 232.19318105839193, 223.83884713053703, 212.51354808919132, 216.61451901309192, 218.94020796753466, 216.79470012895763, 224.55096617341042, 214.89217784255743, 216.46798704750836, 219.12149898707867, 215.7948489766568], "server_ms": [210.58148401789367, 209.1008760035038, 209.59512307308614, 210.76919697225094, 209.47033097036183, 217.24145719781518, 209.20779719017446, 212.1926541440189, 212.15731417760253, 226.65289300493896, 218.10284815728664, 206.90169907175004, 210.98346984945238, 213.38400896638632, 211.12492214888334, 218.8903270289302, 209.339358843863, 210.92559909448028, 213.48508982919157, 210.25720005854964], "statuses": [200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200], "tokens": null, "median_e2e": 216.54125303030014, "median_server": 210.95453447196633}], "tokens": null, "median_e2e_ms": 216.8, "run_medians_e2e_ms": [217.0, 217.0, 216.5], "median_server_ms": 211.1, "non_200": 0}}} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/vram.csv new file mode 100644 index 0000000..008d08d --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r1/vram.csv @@ -0,0 +1,1674 @@ +# pid=1861003 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790785350.100482402,8812,8822 +1790785350.208682233,8812,8822 +1790785350.316793994,8812,8822 +1790785350.424064545,8812,8822 +1790785350.531665690,8812,8822 +1790785350.639092333,8812,8822 +1790785350.748994404,8812,8822 +1790785350.856382796,8812,8822 +1790785350.960278828,8812,8822 +1790785351.062136137,8812,8822 +1790785351.164522832,8812,8822 +1790785351.267017788,8812,8822 +1790785351.373679922,8812,8822 +1790785351.480606539,8812,8822 +1790785351.585254150,8812,8822 +1790785351.692465670,8812,8822 +1790785351.803842998,8812,8822 +1790785351.914029323,8812,8822 +1790785352.016952534,8812,8822 +1790785352.118006793,8812,8822 +1790785352.228370550,8812,8822 +1790785352.338071219,8812,8822 +1790785352.443997844,8812,8822 +1790785352.550569627,8812,8822 +1790785352.660539509,8812,8822 +1790785352.767807270,8812,8822 +1790785352.877129615,8812,8822 +1790785352.983316483,8812,8822 +1790785353.094985545,8818,8828 +1790785353.199283031,8818,8828 +1790785353.303752400,8818,8828 +1790785353.411171763,8818,8828 +1790785353.516347290,8818,8828 +1790785353.621956022,8818,8828 +1790785353.725922235,8818,8828 +1790785353.828583103,8818,8828 +1790785353.931661616,8818,8828 +1790785354.034972461,8818,8828 +1790785354.140480422,8818,8828 +1790785354.243143350,8818,8828 +1790785354.347027822,8818,8828 +1790785354.450034044,8818,8828 +1790785354.550063632,8818,8828 +1790785354.651022910,8818,8828 +1790785354.755213656,8818,8828 +1790785354.856838582,8818,8828 +1790785354.962009889,8818,8828 +1790785355.065551177,8818,8828 +1790785355.169889694,8818,8828 +1790785355.271554041,8818,8828 +1790785355.376428195,8818,8828 +1790785355.480195395,8818,8828 +1790785355.583093576,8818,8828 +1790785355.688507466,8818,8828 +1790785355.791352616,8818,8828 +1790785355.893623380,8834,8844 +1790785355.995637110,8834,8844 +1790785356.097616090,8834,8844 +1790785356.199799353,8834,8844 +1790785356.305048071,8834,8844 +1790785356.409441929,8834,8844 +1790785356.512237169,8834,8844 +1790785356.613623622,8834,8844 +1790785356.719256864,8834,8844 +1790785356.825869457,8834,8844 +1790785356.929979992,8834,8844 +1790785357.033964866,8834,8844 +1790785357.137742746,8834,8844 +1790785357.241889941,8834,8844 +1790785357.344446298,8834,8844 +1790785357.447772304,8834,8844 +1790785357.549785404,8834,8844 +1790785357.653829649,8834,8844 +1790785357.759588872,8834,8844 +1790785357.862248430,8834,8844 +1790785357.963087797,8834,8844 +1790785358.071405540,8834,8844 +1790785358.173599983,8834,8844 +1790785358.274123326,8834,8844 +1790785358.383232979,8834,8844 +1790785358.488817200,8834,8844 +1790785358.591320206,8834,8844 +1790785358.694181627,8834,8844 +1790785358.796636982,8834,8844 +1790785358.902137843,8834,8844 +1790785359.007442341,8834,8844 +1790785359.111750519,8834,8844 +1790785359.217140128,8834,8844 +1790785359.323185595,8834,8844 +1790785359.423145882,8834,8844 +1790785359.529254649,8834,8844 +1790785359.630422790,8834,8844 +1790785359.731164016,8834,8844 +1790785359.831592419,8834,8844 +1790785359.938702288,8834,8844 +1790785360.042031614,8834,8844 +1790785360.144190106,8834,8844 +1790785360.249653096,8834,8844 +1790785360.351019170,8834,8844 +1790785360.453920580,8834,8844 +1790785360.555591367,8834,8844 +1790785360.656074350,8834,8844 +1790785360.764679726,8834,8844 +1790785360.865797357,8834,8844 +1790785360.969904561,8834,8844 +1790785361.073226807,8834,8844 +1790785361.175305208,8834,8844 +1790785361.282177215,8834,8844 +1790785361.384743141,8834,8844 +1790785361.485093673,8834,8844 +1790785361.586912062,8834,8844 +1790785361.691932427,8834,8844 +1790785361.792596122,8834,8844 +1790785361.896443294,8834,8844 +1790785362.000350526,8834,8844 +1790785362.100789878,8834,8844 +1790785362.204081564,8834,8844 +1790785362.309617395,8834,8844 +1790785362.415786343,8834,8844 +1790785362.520428074,8834,8844 +1790785362.619825584,8834,8844 +1790785362.722384311,8834,8844 +1790785362.824736366,8834,8844 +1790785362.927651357,8834,8844 +1790785363.032020335,8834,8844 +1790785363.138053351,8834,8844 +1790785363.239358174,8834,8844 +1790785363.346804397,8834,8844 +1790785363.451509498,8834,8844 +1790785363.551674468,8834,8844 +1790785363.653510377,8834,8844 +1790785363.767661066,8834,8844 +1790785363.875509404,8834,8844 +1790785363.982450821,8834,8844 +1790785364.091214709,8834,8844 +1790785364.198462600,8834,8844 +1790785364.300291448,8834,8844 +1790785364.406416826,8834,8844 +1790785364.511647654,8834,8844 +1790785364.624447008,8834,8844 +1790785364.729167940,8834,8844 +1790785364.831485214,8834,8844 +1790785364.938059637,8834,8844 +1790785365.041979859,8834,8844 +1790785365.144257923,8834,8844 +1790785365.247676190,8834,8844 +1790785365.348506327,8834,8844 +1790785365.451501439,8834,8844 +1790785365.554959046,8834,8844 +1790785365.653750960,8834,8844 +1790785365.762836512,8834,8844 +1790785365.863556587,8834,8844 +1790785365.967132566,8834,8844 +1790785366.071290511,8834,8844 +1790785366.173487164,8834,8844 +1790785366.277670149,8834,8844 +1790785366.382307670,8834,8844 +1790785366.484775156,8834,8844 +1790785366.590767712,8834,8844 +1790785366.695432963,8834,8844 +1790785366.796500633,8834,8844 +1790785366.899789508,8834,8844 +1790785367.003554469,8834,8844 +1790785367.107733175,8834,8844 +1790785367.209418252,8834,8844 +1790785367.311381351,8834,8844 +1790785367.412507732,8834,8844 +1790785367.515752027,8834,8844 +1790785367.617273952,8834,8844 +1790785367.718157369,8854,8864 +1790785367.819423081,8854,8864 +1790785367.921260650,8854,8864 +1790785368.023443243,8854,8864 +1790785368.123819005,8854,8864 +1790785368.225959916,8854,8864 +1790785368.324716400,8854,8864 +1790785368.425292744,8854,8864 +1790785368.525156120,8854,8864 +1790785368.623461948,8854,8864 +1790785368.725222066,8854,8864 +1790785368.829843856,8854,8864 +1790785368.931145219,8854,8864 +1790785369.035453966,8854,8864 +1790785369.140646613,8854,8864 +1790785369.244607516,8854,8864 +1790785369.348093053,8854,8864 +1790785369.453241790,8854,8864 +1790785369.554595683,8854,8864 +1790785369.657879348,8854,8864 +1790785369.757350550,8854,8864 +1790785369.859040066,8854,8864 +1790785369.959131405,8854,8864 +1790785370.061856974,8854,8864 +1790785370.159929519,8854,8864 +1790785370.264985775,8854,8864 +1790785370.365270426,8854,8864 +1790785370.468741443,8854,8864 +1790785370.569980765,8854,8864 +1790785370.673639354,8854,8864 +1790785370.779196756,8854,8864 +1790785370.877646056,8854,8864 +1790785370.977936836,8854,8864 +1790785371.083966553,8854,8864 +1790785371.189025489,8854,8864 +1790785371.292155262,8854,8864 +1790785371.396762793,8854,8864 +1790785371.498519020,8854,8864 +1790785371.597041541,8854,8864 +1790785371.699328414,8854,8864 +1790785371.799729446,8854,8864 +1790785371.901192431,8854,8864 +1790785372.001856096,8854,8864 +1790785372.105531396,8854,8864 +1790785372.210906375,8854,8864 +1790785372.313789886,8854,8864 +1790785372.417536406,8854,8864 +1790785372.522535521,8854,8864 +1790785372.621313765,8854,8864 +1790785372.725540760,8854,8864 +1790785372.829425552,8854,8864 +1790785372.935129925,8854,8864 +1790785373.037557071,8854,8864 +1790785373.139877965,8854,8864 +1790785373.243062429,8854,8864 +1790785373.339923491,8854,8864 +1790785373.444367239,8854,8864 +1790785373.543619428,8854,8864 +1790785373.644795909,8854,8864 +1790785373.743613143,8854,8864 +1790785373.843732822,8854,8864 +1790785373.942965101,8854,8864 +1790785374.043007029,8854,8864 +1790785374.152321013,8854,8864 +1790785374.262490997,8854,8864 +1790785374.371769921,8854,8864 +1790785374.480976054,8854,8864 +1790785374.591957138,8854,8864 +1790785374.701720378,8854,8864 +1790785374.809682846,8854,8864 +1790785374.919838770,8854,8864 +1790785375.029961934,8854,8864 +1790785375.136565288,8854,8864 +1790785375.245030602,8854,8864 +1790785375.353351625,8854,8864 +1790785375.463762532,8854,8864 +1790785375.573285749,8854,8864 +1790785375.681697663,8854,8864 +1790785375.789054475,8854,8864 +1790785375.897727502,8854,8864 +1790785376.005052503,8854,8864 +1790785376.116004827,8854,8864 +1790785376.224640783,8854,8864 +1790785376.333866557,8854,8864 +1790785376.439770732,8854,8864 +1790785376.548697452,8854,8864 +1790785376.657148147,8854,8864 +1790785376.767472053,8854,8864 +1790785376.876022648,8854,8864 +1790785376.985845988,8854,8864 +1790785377.093161100,8854,8864 +1790785377.204197314,8854,8864 +1790785377.313414247,8854,8864 +1790785377.421333826,8854,8864 +1790785377.529806771,8854,8864 +1790785377.646215666,8854,8864 +1790785377.757710666,8854,8864 +1790785377.864673393,8854,8864 +1790785377.973175208,8854,8864 +1790785378.082410112,8854,8864 +1790785378.191760277,8854,8864 +1790785378.301661258,8854,8864 +1790785378.405314007,8854,8864 +1790785378.510339653,8854,8864 +1790785378.616120946,8854,8864 +1790785378.717845394,8854,8864 +1790785378.821495583,8854,8864 +1790785378.925852940,8854,8864 +1790785379.024320880,8854,8864 +1790785379.129631739,8854,8864 +1790785379.233548731,8854,8864 +1790785379.339963213,8854,8864 +1790785379.445559234,8854,8864 +1790785379.551313788,8854,8864 +1790785379.653896895,8854,8864 +1790785379.753076533,8854,8864 +1790785379.857010886,8854,8864 +1790785379.961670807,8854,8864 +1790785380.064754960,8854,8864 +1790785380.168372839,8854,8864 +1790785380.271896397,8854,8864 +1790785380.377442428,8854,8864 +1790785380.476996131,8854,8864 +1790785380.579096512,8854,8864 +1790785380.682763221,8854,8864 +1790785380.787101089,8854,8864 +1790785380.890136461,8854,8864 +1790785380.991993700,8854,8864 +1790785381.097125527,8854,8864 +1790785381.201631836,8854,8864 +1790785381.303263582,8854,8864 +1790785381.413825811,8854,8864 +1790785381.524427340,8854,8864 +1790785381.625920855,8854,8864 +1790785381.724936981,8854,8864 +1790785381.829210978,8854,8864 +1790785381.935186703,8854,8864 +1790785382.038504419,8854,8864 +1790785382.142649164,8854,8864 +1790785382.239735618,8854,8864 +1790785382.342361086,8854,8864 +1790785382.445794073,8854,8864 +1790785382.545817641,8854,8864 +1790785382.649613812,8854,8864 +1790785382.753637286,8854,8864 +1790785382.853286409,8854,8864 +1790785382.959339516,8854,8864 +1790785383.064562474,8854,8864 +1790785383.167120380,8854,8864 +1790785383.277847031,8854,8864 +1790785383.381144106,8854,8864 +1790785383.484790906,8854,8864 +1790785383.585997287,8854,8864 +1790785383.692605960,8854,8864 +1790785383.797413453,8854,8864 +1790785383.899982210,8854,8864 +1790785384.003631460,8854,8864 +1790785384.110995501,8854,8864 +1790785384.211612647,8854,8864 +1790785384.318368764,8854,8864 +1790785384.433067339,8854,8864 +1790785384.542747948,8854,8864 +1790785384.652909293,8854,8864 +1790785384.755184768,8854,8864 +1790785384.853494426,8854,8864 +1790785384.957651501,8854,8864 +1790785385.064923332,8854,8864 +1790785385.164332223,8854,8864 +1790785385.269303997,8854,8864 +1790785385.370329646,8854,8864 +1790785385.471622099,8854,8864 +1790785385.577387313,8854,8864 +1790785385.677462301,8854,8864 +1790785385.778763394,8854,8864 +1790785385.877385705,8854,8864 +1790785385.982866316,8854,8864 +1790785386.091090428,8854,8864 +1790785386.193391852,8854,8864 +1790785386.293092556,8854,8864 +1790785386.395307768,8854,8864 +1790785386.492993509,8854,8864 +1790785386.597838413,8854,8864 +1790785386.704000081,8854,8864 +1790785386.804951410,8854,8864 +1790785386.917350009,8854,8864 +1790785387.020461532,8854,8864 +1790785387.122053008,8854,8864 +1790785387.220022162,8854,8864 +1790785387.321720119,8854,8864 +1790785387.425349539,8854,8864 +1790785387.530424164,8854,8864 +1790785387.633931512,8854,8864 +1790785387.738992088,8854,8864 +1790785387.841681606,8854,8864 +1790785387.947607322,8854,8864 +1790785388.049610622,8854,8864 +1790785388.153584275,8854,8864 +1790785388.257394947,8854,8864 +1790785388.359585049,8854,8864 +1790785388.466494336,8854,8864 +1790785388.570068354,8854,8864 +1790785388.673883976,8854,8864 +1790785388.777149331,8854,8864 +1790785388.881058133,8854,8864 +1790785388.985144328,8854,8864 +1790785389.085981905,8854,8864 +1790785389.192471477,8854,8864 +1790785389.295954304,8854,8864 +1790785389.399521873,8854,8864 +1790785389.500142527,8854,8864 +1790785389.604324193,8854,8864 +1790785389.707278244,8854,8864 +1790785389.809142313,8854,8864 +1790785389.914902127,8854,8864 +1790785390.019753900,8854,8864 +1790785390.123673233,8854,8864 +1790785390.225965136,8854,8864 +1790785390.327833615,8854,8864 +1790785390.432023651,8854,8864 +1790785390.536338738,8854,8864 +1790785390.641989170,8854,8864 +1790785390.750154171,8854,8864 +1790785390.852984812,8854,8864 +1790785390.953376214,8854,8864 +1790785391.057161395,8854,8864 +1790785391.162070978,8854,8864 +1790785391.265909470,8854,8864 +1790785391.367162582,8854,8864 +1790785391.469792070,8854,8864 +1790785391.570394184,8854,8864 +1790785391.670149189,8854,8864 +1790785391.776614070,8854,8864 +1790785391.880661875,8854,8864 +1790785391.983899369,8854,8864 +1790785392.085357783,8854,8864 +1790785392.190000134,8854,8864 +1790785392.293808865,8854,8864 +1790785392.398901271,8854,8864 +1790785392.505905859,8854,8864 +1790785392.609196964,8854,8864 +1790785392.709467936,8854,8864 +1790785392.812041982,8854,8864 +1790785392.917760545,8854,8864 +1790785393.021430765,8854,8864 +1790785393.125442479,8854,8864 +1790785393.229469873,8854,8864 +1790785393.334567148,8854,8864 +1790785393.440019389,8854,8864 +1790785393.544366056,8854,8864 +1790785393.648898285,8854,8864 +1790785393.753685548,8854,8864 +1790785393.854951930,8854,8864 +1790785393.960545672,8854,8864 +1790785394.065393225,8854,8864 +1790785394.168169554,8854,8864 +1790785394.272232269,8854,8864 +1790785394.377705589,8854,8864 +1790785394.477696056,8854,8864 +1790785394.580872260,8854,8864 +1790785394.683514658,8854,8864 +1790785394.788089638,8854,8864 +1790785394.891280182,8854,8864 +1790785394.995391537,8854,8864 +1790785395.098068845,8854,8864 +1790785395.200049236,8854,8864 +1790785395.305708118,8854,8864 +1790785395.409827173,8854,8864 +1790785395.512366079,8854,8864 +1790785395.617477065,8854,8864 +1790785395.721141045,8854,8864 +1790785395.823723432,8854,8864 +1790785395.924997044,8854,8864 +1790785396.029696566,8854,8864 +1790785396.130115508,8854,8864 +1790785396.229938264,8854,8864 +1790785396.331942694,8854,8864 +1790785396.437008570,8854,8864 +1790785396.540557848,8854,8864 +1790785396.640670777,8854,8864 +1790785396.746014606,8854,8864 +1790785396.850006009,8854,8864 +1790785396.955319748,8854,8864 +1790785397.053388663,8854,8864 +1790785397.155591736,8854,8864 +1790785397.260418038,8854,8864 +1790785397.360759330,8854,8864 +1790785397.468180433,8854,8864 +1790785397.569916570,8854,8864 +1790785397.676228200,8854,8864 +1790785397.779560696,8854,8864 +1790785397.881145932,8854,8864 +1790785397.987303580,8854,8864 +1790785398.097153990,8854,8864 +1790785398.205537944,8854,8864 +1790785398.314935620,8854,8864 +1790785398.422454753,8854,8864 +1790785398.530601765,8854,8864 +1790785398.633947670,8854,8864 +1790785398.739229138,8854,8864 +1790785398.839027424,8854,8864 +1790785398.945615827,8854,8864 +1790785399.047347644,8854,8864 +1790785399.148508065,8854,8864 +1790785399.253198056,8854,8864 +1790785399.359222443,8854,8864 +1790785399.461668149,8854,8864 +1790785399.563097132,8854,8864 +1790785399.666002783,8854,8864 +1790785399.769300509,8854,8864 +1790785399.879621405,8854,8864 +1790785399.992436329,8854,8864 +1790785400.100783242,8854,8864 +1790785400.209598671,8854,8864 +1790785400.313402963,8854,8864 +1790785400.417436046,8854,8864 +1790785400.523128709,8854,8864 +1790785400.623591042,8854,8864 +1790785400.729391496,8854,8864 +1790785400.831027412,8854,8864 +1790785400.935989227,8854,8864 +1790785401.038892848,8854,8864 +1790785401.140535804,8854,8864 +1790785401.243244943,8854,8864 +1790785401.344163051,8854,8864 +1790785401.446888989,8854,8864 +1790785401.550736171,8854,8864 +1790785401.652888584,8854,8864 +1790785401.755128857,8854,8864 +1790785401.856822553,8854,8864 +1790785401.961288313,8854,8864 +1790785402.065565319,8854,8864 +1790785402.169451591,8854,8864 +1790785402.276288427,8854,8864 +1790785402.376528987,8854,8864 +1790785402.481735595,8854,8864 +1790785402.585114531,8854,8864 +1790785402.688525068,8854,8864 +1790785402.792666143,8854,8864 +1790785402.897765279,8854,8864 +1790785402.999076811,8854,8864 +1790785403.104773974,8854,8864 +1790785403.207135339,8854,8864 +1790785403.310545416,8854,8864 +1790785403.415595691,8854,8864 +1790785403.520468505,8854,8864 +1790785403.624348496,8854,8864 +1790785403.733063024,8854,8864 +1790785403.836854475,8854,8864 +1790785403.941622887,8854,8864 +1790785404.041850698,8854,8864 +1790785404.147093665,8854,8864 +1790785404.250080887,8854,8864 +1790785404.349715470,8854,8864 +1790785404.454919677,8854,8864 +1790785404.556962378,8854,8864 +1790785404.657960448,8854,8864 +1790785404.762693839,8854,8864 +1790785404.865887943,8854,8864 +1790785404.972405386,8854,8864 +1790785405.077806725,8854,8864 +1790785405.177703442,8854,8864 +1790785405.281998818,8854,8864 +1790785405.383077148,8854,8864 +1790785405.493272493,8854,8864 +1790785405.598840444,8854,8864 +1790785405.702105169,8854,8864 +1790785405.804673926,8854,8864 +1790785405.905131719,8854,8864 +1790785406.007651996,8854,8864 +1790785406.113517050,8854,8864 +1790785406.215151717,8854,8864 +1790785406.319836448,8854,8864 +1790785406.423664569,8854,8864 +1790785406.526374218,8854,8864 +1790785406.625092421,8854,8864 +1790785406.736015664,8854,8864 +1790785406.849367935,8854,8864 +1790785406.959247175,8854,8864 +1790785407.069621562,8854,8864 +1790785407.179164259,8854,8864 +1790785407.288901648,8854,8864 +1790785407.396517343,8854,8864 +1790785407.499254952,8854,8864 +1790785407.599672404,8854,8864 +1790785407.703211152,8854,8864 +1790785407.805676779,8854,8864 +1790785407.913116061,8854,8864 +1790785408.019079347,8854,8864 +1790785408.121689945,8854,8864 +1790785408.224820678,8854,8864 +1790785408.326370024,8854,8864 +1790785408.430730131,8854,8864 +1790785408.531407746,8854,8864 +1790785408.633782391,8854,8864 +1790785408.739483634,8854,8864 +1790785408.845325219,8854,8864 +1790785408.948474922,8854,8864 +1790785409.052290383,8854,8864 +1790785409.153986190,8854,8864 +1790785409.258233077,8854,8864 +1790785409.362121679,8854,8864 +1790785409.464762877,8854,8864 +1790785409.571506512,8854,8864 +1790785409.675835689,8854,8864 +1790785409.779719281,8854,8864 +1790785409.884683556,8854,8864 +1790785409.986225411,8854,8864 +1790785410.088059569,8854,8864 +1790785410.192558908,8854,8864 +1790785410.301895173,8854,8864 +1790785410.412543353,8854,8864 +1790785410.524436317,8854,8864 +1790785410.632912662,8854,8864 +1790785410.740292084,8854,8864 +1790785410.849809850,8854,8864 +1790785410.956940300,8854,8864 +1790785411.071300042,8854,8864 +1790785411.182719641,8854,8864 +1790785411.288963680,8854,8864 +1790785411.397853529,8854,8864 +1790785411.513396765,8854,8864 +1790785411.621907481,8854,8864 +1790785411.732370248,8854,8864 +1790785411.841235858,8854,8864 +1790785411.949582511,8854,8864 +1790785412.059882106,8854,8864 +1790785412.171694999,8854,8864 +1790785412.281365798,8854,8864 +1790785412.393233492,8854,8864 +1790785412.496385235,8854,8864 +1790785412.604950621,8854,8864 +1790785412.711156210,8854,8864 +1790785412.821543747,8854,8864 +1790785412.931452378,8854,8864 +1790785413.039380077,8854,8864 +1790785413.150147778,8854,8864 +1790785413.252303970,8854,8864 +1790785413.356150252,8854,8864 +1790785413.459237475,8854,8864 +1790785413.557869446,8854,8864 +1790785413.659655734,8854,8864 +1790785413.766111546,8854,8864 +1790785413.876834777,8854,8864 +1790785413.985836697,8854,8864 +1790785414.092999177,8854,8864 +1790785414.203751868,8854,8864 +1790785414.313770881,8854,8864 +1790785414.426488704,8854,8864 +1790785414.536967951,8854,8864 +1790785414.644413515,8854,8864 +1790785414.751780877,8854,8864 +1790785414.857263247,8854,8864 +1790785414.957199604,8854,8864 +1790785415.055259599,8854,8864 +1790785415.163765614,8854,8864 +1790785415.270536140,8854,8864 +1790785415.377615008,8854,8864 +1790785415.481383369,8854,8864 +1790785415.585035169,8854,8864 +1790785415.688288083,8854,8864 +1790785415.791785141,8854,8864 +1790785415.891750098,8854,8864 +1790785415.997294619,8854,8864 +1790785416.098821564,8854,8864 +1790785416.201186299,8854,8864 +1790785416.301947035,8854,8864 +1790785416.405577174,8854,8864 +1790785416.509951472,8854,8864 +1790785416.613163276,8854,8864 +1790785416.714071525,8854,8864 +1790785416.816283528,8854,8864 +1790785416.924416448,8854,8864 +1790785417.030111271,8854,8864 +1790785417.134149175,8854,8864 +1790785417.235481238,8854,8864 +1790785417.337802752,8854,8864 +1790785417.442973719,8854,8864 +1790785417.541129335,8854,8864 +1790785417.641053342,8854,8864 +1790785417.741777058,8854,8864 +1790785417.845087963,8854,8864 +1790785417.949302699,8854,8864 +1790785418.051599903,8854,8864 +1790785418.157971424,8854,8864 +1790785418.260173076,8854,8864 +1790785418.366077682,8854,8864 +1790785418.469436098,8854,8864 +1790785418.569896821,8854,8864 +1790785418.672523078,8854,8864 +1790785418.775591231,8854,8864 +1790785418.879055578,8854,8864 +1790785418.981866488,8854,8864 +1790785419.090039829,8854,8864 +1790785419.190529143,8854,8864 +1790785419.296560159,8854,8864 +1790785419.400196139,8854,8864 +1790785419.502167489,8854,8864 +1790785419.608885543,8854,8864 +1790785419.709626619,8854,8864 +1790785419.813926856,8854,8864 +1790785419.918003181,8854,8864 +1790785420.021501038,8854,8864 +1790785420.125765685,8854,8864 +1790785420.227075587,8854,8864 +1790785420.332114713,8854,8864 +1790785420.435096124,8854,8864 +1790785420.541497116,8854,8864 +1790785420.645203685,8854,8864 +1790785420.749983588,8854,8864 +1790785420.851690575,8854,8864 +1790785420.954113370,8854,8864 +1790785421.057462706,8854,8864 +1790785421.157116639,8854,8864 +1790785421.270733603,8854,8864 +1790785421.381891109,8854,8864 +1790785421.490793129,8854,8864 +1790785421.590899617,8854,8864 +1790785421.690792224,8854,8864 +1790785421.794095279,8854,8864 +1790785421.897663838,8854,8864 +1790785421.999818860,8854,8864 +1790785422.101502887,8854,8864 +1790785422.203774760,8854,8864 +1790785422.308983667,8854,8864 +1790785422.412515576,8854,8864 +1790785422.512831997,8854,8864 +1790785422.617099323,8854,8864 +1790785422.721966227,8854,8864 +1790785422.825238742,8854,8864 +1790785422.931158198,8854,8864 +1790785423.033483332,8854,8864 +1790785423.145337765,8854,8864 +1790785423.253415806,8854,8864 +1790785423.364047625,8854,8864 +1790785423.474807006,8854,8864 +1790785423.581904625,8854,8864 +1790785423.691556803,8854,8864 +1790785423.801333883,8854,8864 +1790785423.910100271,8854,8864 +1790785424.009808495,8854,8864 +1790785424.114668078,8854,8864 +1790785424.216806520,8854,8864 +1790785424.319025784,8854,8864 +1790785424.419610628,8854,8864 +1790785424.522215935,8854,8864 +1790785424.624019053,8854,8864 +1790785424.729221411,8854,8864 +1790785424.832706508,8854,8864 +1790785424.937712893,8854,8864 +1790785425.044810182,8854,8864 +1790785425.149425612,8854,8864 +1790785425.248990995,8854,8864 +1790785425.349409118,8854,8864 +1790785425.455031549,8854,8864 +1790785425.554531081,8854,8864 +1790785425.659136162,8854,8864 +1790785425.761676428,8854,8864 +1790785425.866991837,8854,8864 +1790785425.967330678,8854,8864 +1790785426.069618512,8854,8864 +1790785426.169229585,8854,8864 +1790785426.271704291,8854,8864 +1790785426.375082337,8854,8864 +1790785426.478201601,8854,8864 +1790785426.581247783,8854,8864 +1790785426.685028254,8854,8864 +1790785426.788617223,8854,8864 +1790785426.888517619,8854,8864 +1790785426.990708872,8854,8864 +1790785427.093430131,8854,8864 +1790785427.195034766,8854,8864 +1790785427.296014705,8854,8864 +1790785427.401881510,8854,8864 +1790785427.501437063,8854,8864 +1790785427.604468165,8854,8864 +1790785427.711301811,8854,8864 +1790785427.813606945,8854,8864 +1790785427.920072087,8854,8864 +1790785428.024093410,8854,8864 +1790785428.133755029,8854,8864 +1790785428.233841097,8854,8864 +1790785428.336417104,8854,8864 +1790785428.438909930,8854,8864 +1790785428.539222502,8854,8864 +1790785428.641465445,8854,8864 +1790785428.747898896,8854,8864 +1790785428.851068930,8854,8864 +1790785428.953477955,8854,8864 +1790785429.059333420,8854,8864 +1790785429.162706556,8854,8864 +1790785429.268863184,8854,8864 +1790785429.372330952,8854,8864 +1790785429.475761308,8854,8864 +1790785429.580155607,8854,8864 +1790785429.687550908,8854,8864 +1790785429.788796431,8854,8864 +1790785429.895512475,8854,8864 +1790785430.000647832,8854,8864 +1790785430.103649244,8854,8864 +1790785430.209541569,8854,8864 +1790785430.309818770,8854,8864 +1790785430.414183977,8854,8864 +1790785430.516919966,8854,8864 +1790785430.621715519,8854,8864 +1790785430.728229331,8854,8864 +1790785430.831722649,8854,8864 +1790785430.936769664,8854,8864 +1790785431.043346247,8854,8864 +1790785431.146951326,8854,8864 +1790785431.250902759,8854,8864 +1790785431.352449054,8854,8864 +1790785431.452472892,8854,8864 +1790785431.554037297,8854,8864 +1790785431.657151930,8854,8864 +1790785431.771946818,8854,8864 +1790785431.873846807,8854,8864 +1790785431.975829827,8854,8864 +1790785432.090146539,8854,8864 +1790785432.197474951,8854,8864 +1790785432.305908595,8854,8864 +1790785432.415303461,8854,8864 +1790785432.517647165,8854,8864 +1790785432.618153678,8854,8864 +1790785432.722781389,8854,8864 +1790785432.825589499,8854,8864 +1790785432.927898503,8854,8864 +1790785433.034050811,8854,8864 +1790785433.137538408,8854,8864 +1790785433.238092742,8854,8864 +1790785433.338814528,8854,8864 +1790785433.440539175,8854,8864 +1790785433.546780324,8854,8864 +1790785433.648906116,8854,8864 +1790785433.752488825,8854,8864 +1790785433.855765340,8854,8864 +1790785433.958733172,8854,8864 +1790785434.063316882,8854,8864 +1790785434.169245647,8854,8864 +1790785434.272831806,8854,8864 +1790785434.370836391,8854,8864 +1790785434.471659428,8854,8864 +1790785434.579005750,8854,8864 +1790785434.682044302,8854,8864 +1790785434.787258759,8854,8864 +1790785434.889994418,8854,8864 +1790785434.992717577,8854,8864 +1790785435.096477737,8854,8864 +1790785435.196976131,8854,8864 +1790785435.306808421,8854,8864 +1790785435.415139394,8854,8864 +1790785435.516786691,8854,8864 +1790785435.618049103,8854,8864 +1790785435.721856974,8854,8864 +1790785435.821641559,8854,8864 +1790785435.923418627,8854,8864 +1790785436.029738957,8854,8864 +1790785436.131851519,8854,8864 +1790785436.235094343,8854,8864 +1790785436.334959289,8854,8864 +1790785436.439777923,8854,8864 +1790785436.545691408,8854,8864 +1790785436.651193999,8854,8864 +1790785436.752221078,8854,8864 +1790785436.851086502,8854,8864 +1790785436.957799967,8854,8864 +1790785437.060654557,8854,8864 +1790785437.166025356,8854,8864 +1790785437.263744537,8854,8864 +1790785437.369265139,8854,8864 +1790785437.469171115,8854,8864 +1790785437.572081346,8854,8864 +1790785437.674609252,8854,8864 +1790785437.776962237,8854,8864 +1790785437.881206713,8854,8864 +1790785437.986834055,8854,8864 +1790785438.088034667,8854,8864 +1790785438.189985686,8854,8864 +1790785438.292833257,8854,8864 +1790785438.396603147,8854,8864 +1790785438.501265788,8854,8864 +1790785438.603192548,8854,8864 +1790785438.704008475,8854,8864 +1790785438.807851616,8854,8864 +1790785438.909141449,8854,8864 +1790785439.011376122,8854,8864 +1790785439.113960629,8854,8864 +1790785439.215241061,8854,8864 +1790785439.317555825,8854,8864 +1790785439.423741404,8854,8864 +1790785439.526938638,8854,8864 +1790785439.629833779,8854,8864 +1790785439.728197788,8854,8864 +1790785439.831810007,8854,8864 +1790785439.932409111,8854,8864 +1790785440.035944429,8854,8864 +1790785440.137792478,8854,8864 +1790785440.242062474,8854,8864 +1790785440.344873425,8854,8864 +1790785440.448243301,8854,8864 +1790785440.549111748,8854,8864 +1790785440.644538653,8854,8864 +1790785440.749742300,8854,8864 +1790785440.857029021,8854,8864 +1790785440.961526600,8854,8864 +1790785441.064835556,8854,8864 +1790785441.168556856,8854,8864 +1790785441.272197886,8854,8864 +1790785441.376912857,8854,8864 +1790785441.481943582,8854,8864 +1790785441.583847062,8854,8864 +1790785441.684108143,8854,8864 +1790785441.788043475,8854,8864 +1790785441.893789419,8854,8864 +1790785441.997801362,8854,8864 +1790785442.105185965,8854,8864 +1790785442.207536479,8854,8864 +1790785442.313673177,8854,8864 +1790785442.416043442,8854,8864 +1790785442.522000887,8854,8864 +1790785442.625865159,8854,8864 +1790785442.732153059,8854,8864 +1790785442.834808517,8854,8864 +1790785442.937416815,8854,8864 +1790785443.038819228,8854,8864 +1790785443.138936807,8854,8864 +1790785443.245599301,8854,8864 +1790785443.359780111,8854,8864 +1790785443.461169795,8854,8864 +1790785443.559072798,8854,8864 +1790785443.667133718,8854,8864 +1790785443.771551466,8854,8864 +1790785443.876963716,8854,8864 +1790785443.977047255,8854,8864 +1790785444.081408912,8854,8864 +1790785444.184992811,8854,8864 +1790785444.287805031,8854,8864 +1790785444.392776765,8854,8864 +1790785444.500591612,8854,8864 +1790785444.602009306,8854,8864 +1790785444.702128215,8854,8864 +1790785444.809026212,8854,8864 +1790785444.913639212,8854,8864 +1790785445.017518734,8854,8864 +1790785445.118966298,8854,8864 +1790785445.215260473,8854,8864 +1790785445.317648908,8854,8864 +1790785445.423808877,8854,8864 +1790785445.528558108,8854,8864 +1790785445.631766143,8854,8916 +1790785445.730390025,8928,8938 +1790785445.837689596,8928,8938 +1790785445.941443167,8928,8938 +1790785446.047508773,8928,8938 +1790785446.147280759,8928,8938 +1790785446.247161975,8928,8938 +1790785446.351238639,8928,8938 +1790785446.455354724,8928,8938 +1790785446.561306140,8928,8938 +1790785446.663569873,8928,8938 +1790785446.763434309,8928,8938 +1790785446.863753290,8928,8938 +1790785446.964585328,8928,8938 +1790785447.066004091,8928,8938 +1790785447.175264685,8928,8938 +1790785447.284921823,8928,8938 +1790785447.393688391,8928,8938 +1790785447.504445612,8928,8938 +1790785447.613171690,8928,8938 +1790785447.723460906,8928,8938 +1790785447.830310632,8928,8938 +1790785447.937583613,8928,8938 +1790785448.044654481,8928,8938 +1790785448.152205256,8928,8938 +1790785448.260353317,8928,8938 +1790785448.368961433,8928,8938 +1790785448.480711135,8928,8938 +1790785448.589205670,8928,8938 +1790785448.696830666,8928,8938 +1790785448.803693941,8928,8938 +1790785448.908309392,8928,8938 +1790785449.011908351,8964,8974 +1790785449.117859607,8964,8974 +1790785449.217898465,8964,8974 +1790785449.329564996,8964,8974 +1790785449.439559438,8964,8974 +1790785449.542170976,8964,8974 +1790785449.641762579,8964,8974 +1790785449.743460526,8964,8974 +1790785449.845562047,8964,8974 +1790785449.952603996,8964,8974 +1790785450.053871468,8964,8974 +1790785450.157928052,8964,8974 +1790785450.259532888,8964,8974 +1790785450.362093825,8964,8974 +1790785450.463982434,8964,8974 +1790785450.567096797,8964,8974 +1790785450.679444927,8964,8974 +1790785450.786701177,8964,8974 +1790785450.894575735,8964,8974 +1790785451.001472822,8964,8974 +1790785451.112414515,8964,8974 +1790785451.220864729,8964,8974 +1790785451.333266649,8964,8974 +1790785451.443659416,8964,8974 +1790785451.553044121,8964,8974 +1790785451.661365694,8964,8974 +1790785451.772394989,8964,8974 +1790785451.878188532,8964,8974 +1790785451.983601932,8964,8974 +1790785452.089789600,8964,8974 +1790785452.190883681,8964,8974 +1790785452.292592238,8964,8974 +1790785452.397910726,8964,8974 +1790785452.498551291,8964,8974 +1790785452.602153570,8964,8974 +1790785452.705588447,8964,8974 +1790785452.807820950,8964,8974 +1790785452.914370033,8964,8974 +1790785453.017234623,8964,8974 +1790785453.126240734,8964,8974 +1790785453.228903372,8964,8974 +1790785453.332885236,8964,8974 +1790785453.436409743,8964,8974 +1790785453.541911884,8964,8974 +1790785453.645769916,8964,8974 +1790785453.749952871,8964,8974 +1790785453.854068306,8964,8974 +1790785453.958859458,8964,8974 +1790785454.061909881,8964,8974 +1790785454.165860714,8964,9182 +1790785454.279863732,9172,9438 +1790785454.389847964,9428,9438 +1790785454.500690116,9428,9438 +1790785454.604021682,9428,9438 +1790785454.707017964,9428,9438 +1790785454.815059604,9428,9438 +1790785454.916118743,9428,9438 +1790785455.021074898,9428,9438 +1790785455.123757226,9428,9438 +1790785455.237057896,9428,9438 +1790785455.337991624,9428,9438 +1790785455.436588233,9428,9438 +1790785455.537231783,9428,9438 +1790785455.635467374,9428,9438 +1790785455.736659520,9428,9438 +1790785455.834611987,9428,9438 +1790785455.936776286,9428,9438 +1790785456.043392199,9430,9440 +1790785456.148144400,9430,9440 +1790785456.257970253,9430,9440 +1790785456.360232213,9430,9440 +1790785456.475533883,9430,9440 +1790785456.577467429,9430,9440 +1790785456.687818699,9430,9440 +1790785456.791745899,9430,9440 +1790785456.893100808,9430,9440 +1790785457.007163643,9430,9440 +1790785457.109818778,9430,9440 +1790785457.210035062,9430,9440 +1790785457.309871722,9430,9440 +1790785457.418671853,9430,9440 +1790785457.527388652,9430,9440 +1790785457.638724034,9430,9440 +1790785457.745939565,9430,9440 +1790785457.852261865,9430,9440 +1790785457.955552777,9430,9440 +1790785458.059507827,9430,9440 +1790785458.165856728,9430,9440 +1790785458.273458694,9430,9440 +1790785458.376908778,9430,9440 +1790785458.494780890,9430,9440 +1790785458.599218367,9430,9440 +1790785458.712528803,9430,9440 +1790785458.819639512,9430,9440 +1790785458.927944117,9430,9440 +1790785459.033177603,9430,9440 +1790785459.137963784,9430,9440 +1790785459.244964722,9430,9440 +1790785459.349900535,9430,9440 +1790785459.456501628,9430,9440 +1790785459.567970192,9430,9440 +1790785459.677596153,9430,9440 +1790785459.778112661,9430,9440 +1790785459.893773796,9430,9440 +1790785459.999114504,9430,9440 +1790785460.105993290,9430,9440 +1790785460.212448792,9430,9440 +1790785460.321991881,9430,9440 +1790785460.429399345,9430,9440 +1790785460.536796087,9430,9440 +1790785460.636163592,9430,9440 +1790785460.745649091,9614,9624 +1790785460.858980147,9614,9120 +1790785460.967832878,9614,9624 +1790785461.072849002,9614,9624 +1790785461.178725956,9614,9624 +1790785461.282500024,8814,9624 +1790785461.384820095,9614,9624 +1790785461.493069669,8982,9624 +1790785461.601177731,9614,9624 +1790785461.704764317,9614,9624 +1790785461.809935373,9614,8824 +1790785461.915031857,9614,9624 +1790785462.028153941,9614,9624 +1790785462.140767008,9614,9624 +1790785462.248869190,8814,9624 +1790785462.361799012,9614,9624 +1790785462.466778885,9614,9624 +1790785462.572021202,9614,8828 +1790785462.673477291,9614,9624 +1790785462.779914832,9614,9184 +1790785462.876376171,9614,9624 +1790785462.985016549,9614,9624 +1790785463.089633879,9614,9624 +1790785463.192605537,9562,9624 +1790785463.297926785,9614,9624 +1790785463.401277288,8814,9624 +1790785463.508099874,9614,9624 +1790785463.615005061,9614,9624 +1790785463.720907376,9614,9624 +1790785463.823961405,9614,9624 +1790785463.931418519,9614,8824 +1790785464.037511426,9614,9624 +1790785464.140671887,9614,9624 +1790785464.243974930,9614,9624 +1790785464.356561507,9562,9624 +1790785464.460911962,9614,9624 +1790785464.564772932,8878,9624 +1790785464.671187783,9614,9624 +1790785464.773972129,9614,9624 +1790785464.876516312,9614,9624 +1790785464.982759202,9614,9624 +1790785465.085406326,9614,8824 +1790785465.189107754,9614,9624 +1790785465.291638417,9614,9624 +1790785465.397789365,9614,9624 +1790785465.505928277,9614,9624 +1790785465.607684021,9614,9624 +1790785465.715108894,8814,9624 +1790785465.819028904,9614,9624 +1790785465.933846229,9614,9624 +1790785466.043666492,9614,8824 +1790785466.153087120,9614,9624 +1790785466.262012222,9614,9624 +1790785466.375146355,9614,9624 +1790785466.487783653,8836,9624 +1790785466.597392114,9614,9624 +1790785466.709204011,9614,9624 +1790785466.819788913,9614,8846 +1790785466.929553486,9614,9624 +1790785467.037655918,9614,9624 +1790785467.147771124,9614,9624 +1790785467.252936500,8814,9624 +1790785467.355443073,9614,9624 +1790785467.461983855,9614,9624 +1790785467.566707515,9614,9624 +1790785467.675627057,9614,9624 +1790785467.783459206,9614,8824 +1790785467.893430511,9614,9624 +1790785467.999792741,9614,9624 +1790785468.104938026,9614,9624 +1790785468.209665897,8962,9624 +1790785468.315548621,9614,9624 +1790785468.419712905,8836,9624 +1790785468.522637402,9614,9624 +1790785468.630508372,9614,9624 +1790785468.736934982,9614,9624 +1790785468.843517066,9614,9624 +1790785468.949915716,9614,9056 +1790785469.055632289,9614,9624 +1790785469.159829713,9614,9624 +1790785469.264030896,9614,9624 +1790785469.369450825,9466,9624 +1790785469.472615526,9614,9624 +1790785469.576811690,8814,9624 +1790785469.681434899,9614,9624 +1790785469.786267120,9614,9624 +1790785469.888572971,9614,9624 +1790785469.996386709,9614,9624 +1790785470.099753603,9614,8824 +1790785470.200005448,9614,9624 +1790785470.302935595,9614,9184 +1790785470.407461453,9614,9624 +1790785470.513621401,9614,9624 +1790785470.620938133,9614,9624 +1790785470.727295424,8814,9624 +1790785470.835343145,9614,9624 +1790785470.941773786,9614,9624 +1790785471.044186768,9614,9624 +1790785471.149130571,9614,9624 +1790785471.253463646,9614,8824 +1790785471.357784861,9614,9624 +1790785471.469606859,9614,9624 +1790785471.576320014,9614,9624 +1790785471.683447024,8814,9624 +1790785471.789777073,9614,9624 +1790785471.896274696,9614,9624 +1790785471.997697875,9614,9624 +1790785472.102857561,9614,9624 +1790785472.207794143,9614,8824 +1790785472.313580836,9614,9624 +1790785472.415860287,9614,9160 +1790785472.519456403,9150,9416 +1790785472.631582184,9406,9416 +1790785472.729702883,9406,9416 +1790785472.843942631,9406,9416 +1790785472.947328524,9406,9416 +1790785473.048151587,9406,9416 +1790785473.153204471,9406,9416 +1790785473.252258191,9406,9416 +1790785473.357003941,9406,9416 +1790785473.463746347,9406,9416 +1790785473.572893721,9406,9416 +1790785473.676169444,9406,9416 +1790785473.775666039,9406,9416 +1790785473.879952334,9406,9416 +1790785473.981507925,9406,9416 +1790785474.085634858,9406,9416 +1790785474.182745935,9406,9416 +1790785474.290822426,9406,9416 +1790785474.394069438,9406,9416 +1790785474.503804820,9406,9416 +1790785474.605764216,9406,9416 +1790785474.718966501,9406,9416 +1790785474.827910873,9406,9416 +1790785474.943544478,9406,9416 +1790785475.048197837,9406,9416 +1790785475.161029857,9406,9416 +1790785475.273327000,9406,9416 +1790785475.378339544,9406,9416 +1790785475.485310962,9406,9416 +1790785475.603327066,9406,9416 +1790785475.716771064,9406,9416 +1790785475.830846849,9406,9416 +1790785475.937059877,9406,9416 +1790785476.043470189,9406,9416 +1790785476.152491442,9406,9416 +1790785476.249814181,9406,9416 +1790785476.356718227,9406,9416 +1790785476.465581509,9406,9416 +1790785476.565989966,9406,9416 +1790785476.675436834,9406,9416 +1790785476.776968095,9406,9416 +1790785476.885852647,9406,9416 +1790785476.990479106,9406,9416 +1790785477.101008267,9406,9416 +1790785477.207672872,9406,9416 +1790785477.312943768,9406,9416 +1790785477.423865405,9406,9416 +1790785477.526186555,9406,9416 +1790785477.632529826,9406,9416 +1790785477.736486327,9406,9416 +1790785477.839117551,9406,9416 +1790785477.936869675,9406,9416 +1790785478.044381740,9406,9416 +1790785478.149198591,9406,9416 +1790785478.257874340,9406,9416 +1790785478.363685373,9406,9416 +1790785478.471459031,9406,9416 +1790785478.580957350,9406,9416 +1790785478.687715985,9406,9490 +1790785478.793445118,9480,9490 +1790785478.895382734,8814,9732 +1790785479.004102753,9722,9732 +1790785479.111144922,8814,9732 +1790785479.214222182,9722,9732 +1790785479.323166454,9400,9732 +1790785479.426422936,9722,9732 +1790785479.530135344,9722,9732 +1790785479.637351895,9722,9732 +1790785479.743937418,9722,9486 +1790785479.849003922,9722,9732 +1790785479.949747603,9722,8824 +1790785480.052863124,9722,9732 +1790785480.157599654,9722,8824 +1790785480.262423096,9722,9732 +1790785480.368640684,9722,9708 +1790785480.474668511,9722,9732 +1790785480.577956273,9722,9732 +1790785480.684805830,9722,9732 +1790785480.792932981,9722,9732 +1790785480.893419300,9722,9732 +1790785480.999678268,9722,9732 +1790785481.105649254,9722,9732 +1790785481.211200134,9722,9732 +1790785481.317869089,9078,9732 +1790785481.420569624,9722,9732 +1790785481.525764410,8814,9732 +1790785481.629750591,9722,9732 +1790785481.733124625,9518,9732 +1790785481.839612946,9722,9732 +1790785481.944506239,9722,9732 +1790785482.048886865,9722,9732 +1790785482.157506503,9722,9732 +1790785482.256804336,9722,9732 +1790785482.363985277,9722,9270 +1790785482.469171103,9722,9732 +1790785482.572476095,9722,8824 +1790785482.680461696,9722,9732 +1790785482.784625039,9722,8824 +1790785482.892041872,9722,9732 +1790785482.993473962,9722,9076 +1790785483.096557742,9722,9732 +1790785483.203556910,9722,9732 +1790785483.305834470,9722,9732 +1790785483.411218418,9722,9732 +1790785483.520264672,9722,9732 +1790785483.626041295,9722,9732 +1790785483.727623327,9722,9732 +1790785483.835188992,9722,9732 +1790785483.939319025,9078,9732 +1790785484.043672080,9722,9732 +1790785484.146799021,8814,9732 +1790785484.256504133,9722,9732 +1790785484.361249103,9482,9732 +1790785484.469409866,9722,9732 +1790785484.574824534,9722,9732 +1790785484.683274190,9722,9732 +1790785484.793984154,9722,9732 +1790785484.896995073,9722,9732 +1790785485.003047580,9722,9732 +1790785485.108498579,9722,9732 +1790785485.213454552,9722,9732 +1790785485.320682043,9722,9732 +1790785485.425544255,9722,8942 +1790785485.529428105,9722,9732 +1790785485.632682187,9722,8824 +1790785485.736569047,9722,9732 +1790785485.841942895,9722,8824 +1790785485.944093834,9722,9732 +1790785486.043831052,9722,9732 +1790785486.147604711,9722,9732 +1790785486.257052169,9722,9732 +1790785486.359600603,9722,9732 +1790785486.459606925,9722,9732 +1790785486.563126330,8838,9732 +1790785486.665608453,9722,9732 +1790785486.773070446,9066,9732 +1790785486.878027080,9722,9732 +1790785486.982058391,9722,9732 +1790785487.085683648,9722,9732 +1790785487.191015515,9722,9732 +1790785487.297836822,9722,9732 +1790785487.401466218,9722,9732 +1790785487.504853282,9722,9732 +1790785487.612694850,9722,8824 +1790785487.715173253,9722,9732 +1790785487.816852406,9722,8824 +1790785487.929971609,9722,9732 +1790785488.040363519,9722,8824 +1790785488.148948287,9722,9732 +1790785488.255538330,9722,8824 +1790785488.365621436,9722,9732 +1790785488.469092881,9722,8824 +1790785488.575406031,9722,9732 +1790785488.681846442,9722,8824 +1790785488.789524119,9722,9732 +1790785488.896679429,9722,8824 +1790785489.003289823,9722,9732 +1790785489.109349599,9722,9732 +1790785489.212383059,9722,9732 +1790785489.319584479,9722,9732 +1790785489.423655592,9722,9732 +1790785489.529139741,9722,9732 +1790785489.635320309,9722,9732 +1790785489.737648420,9722,9732 +1790785489.844109161,9114,9732 +1790785489.949821663,9722,9732 +1790785490.056402147,8814,9732 +1790785490.161936787,9722,9732 +1790785490.264453820,9518,9732 +1790785490.369229121,9722,9732 +1790785490.476176498,9722,9732 +1790785490.581582587,9722,9732 +1790785490.692049558,9722,9732 +1790785490.798050214,9722,9732 +1790785490.901037323,9722,9732 +1790785491.004637169,9722,9732 +1790785491.111564616,9722,8824 +1790785491.213218428,9722,9732 +1790785491.312618283,9722,9148 +1790785491.419929215,9722,9732 +1790785491.525348744,9722,9732 +1790785491.633367055,9722,9732 +1790785491.741674909,9722,9732 +1790785491.845617329,9722,9732 +1790785491.948322095,9722,9732 +1790785492.057077055,9722,9732 +1790785492.159827671,9722,9732 +1790785492.263644220,8814,8824 +1790785492.365765798,8820,9208 +1790785492.470060193,9198,9208 +1790785492.571910777,9198,9208 +1790785492.674436741,9198,9208 +1790785492.776710491,9198,9660 +1790785492.881749075,10040,10050 +1790785492.986049920,10040,9086 +1790785493.091639871,9722,9732 +1790785493.198084722,9722,8824 +1790785493.301073731,9722,9732 +1790785493.405561018,9722,8824 +1790785493.511439783,9722,9732 +1790785493.618006785,9722,8824 +1790785493.721317278,9722,9732 +1790785493.827042931,9722,8824 +1790785493.929240239,9722,9732 +1790785494.033993590,9722,9732 +1790785494.136333131,9722,9732 +1790785494.241702789,9722,9732 +1790785494.347162178,9722,9732 +1790785494.454226078,9722,9732 +1790785494.561940185,9722,9732 +1790785494.668470127,9722,9732 +1790785494.773792835,9722,9732 +1790785494.875578589,9722,9732 +1790785494.981056348,9722,9732 +1790785495.089298721,9722,9732 +1790785495.193345924,9722,9732 +1790785495.300306021,9722,9732 +1790785495.407180898,9722,9732 +1790785495.512165551,9722,9732 +1790785495.619787427,9722,9732 +1790785495.725565190,9722,9732 +1790785495.829434510,9722,9732 +1790785495.937700174,9722,9732 +1790785496.040096755,9722,9732 +1790785496.143461619,9722,9732 +1790785496.252470312,9722,9732 +1790785496.357398014,9722,9732 +1790785496.464181380,9722,9732 +1790785496.569232914,9722,9732 +1790785496.676688138,9722,9732 +1790785496.781518930,9722,9732 +1790785496.885347499,9722,9732 +1790785496.991793950,9722,9732 +1790785497.100423238,9722,9732 +1790785497.207483848,9722,9732 +1790785497.313042878,9722,9732 +1790785497.420133477,9722,9732 +1790785497.527568531,9722,9732 +1790785497.630716472,9722,9732 +1790785497.733599319,9722,9732 +1790785497.838085277,9722,9732 +1790785497.945605691,9188,9732 +1790785498.050057318,9722,9732 +1790785498.155824781,8814,9732 +1790785498.263112723,9722,9732 +1790785498.366313714,9282,9732 +1790785498.473273982,9722,9732 +1790785498.580521163,9722,9732 +1790785498.685047631,9722,9732 +1790785498.793204364,9722,9732 +1790785498.897054153,9722,9732 +1790785499.007633215,9722,9732 +1790785499.111427394,9722,9732 +1790785499.217394849,9722,9732 +1790785499.323467016,9722,9732 +1790785499.430288672,9722,9732 +1790785499.536490450,9722,9732 +1790785499.641961730,9722,8824 +1790785499.745932531,8936,9410 +1790785499.853027810,9400,9666 +1790785499.960561965,9656,9666 +1790785500.070077074,9656,9666 +1790785500.176528926,9656,9666 +1790785500.278302549,9656,9666 +1790785500.388650499,9656,9666 +1790785500.494584114,9656,9666 +1790785500.594144331,9656,9666 +1790785500.697221081,9656,9666 +1790785500.804679865,9656,9666 +1790785500.908119209,9656,9666 +1790785501.009486058,9656,9666 +1790785501.109342738,9656,9666 +1790785501.210149290,9656,9666 +1790785501.312059425,9656,9666 +1790785501.413161991,9656,9666 +1790785501.518005463,9656,9666 +1790785501.627116637,9656,9666 +1790785501.737276144,9656,9666 +1790785501.843179129,9656,9666 +1790785501.952046160,9656,9666 +1790785502.052613299,9656,9666 +1790785502.162161229,9656,9666 +1790785502.268832693,9656,9666 +1790785502.371482778,9656,9666 +1790785502.483575689,9656,9666 +1790785502.581883330,9656,9666 +1790785502.685657168,9656,9666 +1790785502.785930993,9656,9666 +1790785502.895872088,9656,9666 +1790785503.007291891,9656,9666 +1790785503.117557979,9656,9666 +1790785503.223499154,9656,9666 +1790785503.329794484,9656,9666 +1790785503.444968423,9656,9666 +1790785503.545589632,9656,9666 +1790785503.657139417,9656,9666 +1790785503.763232553,9656,9666 +1790785503.865626405,9656,9666 +1790785503.977820147,9656,9666 +1790785504.087891303,9656,9666 +1790785504.194338465,9656,9666 +1790785504.302997904,9656,9666 +1790785504.408116719,9656,9666 +1790785504.522122163,9656,9666 +1790785504.620423234,9656,9666 +1790785504.736552985,9656,9666 +1790785504.840934531,9656,9666 +1790785504.957819351,9656,9666 +1790785505.062029235,9656,9666 +1790785505.172685308,9656,9666 +1790785505.281590900,9656,9666 +1790785505.393554899,9656,9666 +1790785505.497280847,9656,9666 +1790785505.604861142,9656,9666 +1790785505.717494160,9656,9666 +1790785505.817852566,9656,9666 +1790785505.933772864,9656,9666 +1790785506.040551520,9960,9970 +1790785506.147697230,9960,9970 +1790785506.257124318,9960,9970 +1790785506.361625415,9928,8824 +1790785506.469740098,8856,10056 +1790785506.574067453,10046,10056 +1790785506.681422205,10046,10056 +1790785506.787743106,10046,10056 +1790785506.894745673,9340,10056 +1790785506.998657534,10046,10056 +1790785507.101410880,10046,10056 +1790785507.205288190,10046,10056 +1790785507.309272921,8814,9378 +1790785507.413854709,10046,10056 +1790785507.518522489,10046,10056 +1790785507.623529872,10046,10056 +1790785507.730208906,9782,9378 +1790785507.834867616,10046,10056 +1790785507.937490550,10046,10056 +1790785508.043279064,10046,10056 +1790785508.141485274,10046,8824 +1790785508.247757522,10046,10056 +1790785508.353811059,10046,10056 +1790785508.461516136,10046,10056 +1790785508.565769841,10046,8824 +1790785508.665162285,8814,8824 +1790785508.769142626,9270,10046 +1790785508.875541927,10036,10046 +1790785508.978623307,10036,10046 +1790785509.082905152,10036,10046 +1790785509.191597771,9336,10046 +1790785509.297909780,10036,10046 +1790785509.399478592,10036,10046 +1790785509.502571402,10036,10046 +1790785509.611878459,10036,10046 +1790785509.717191066,10036,10046 +1790785509.821758454,10036,10046 +1790785509.929225658,10036,8824 +1790785510.037223999,10036,10046 +1790785510.144766673,10036,10046 +1790785510.249780197,10036,10046 +1790785510.350796482,10036,8824 +1790785510.456485144,8814,10056 +1790785510.560530286,10046,10056 +1790785510.663320742,10046,10056 +1790785510.767147241,10046,10056 +1790785510.871221284,8814,10056 +1790785510.972876456,10046,10056 +1790785511.079857854,10046,10056 +1790785511.186041472,10046,10056 +1790785511.288600505,8814,10056 +1790785511.394652572,10046,10056 +1790785511.505468938,10046,10056 +1790785511.612542186,10046,10056 +1790785511.721374987,9742,10056 +1790785511.829012883,10046,10056 +1790785511.932864603,10046,10056 +1790785512.037065106,10046,10056 +1790785512.143137173,8814,10056 +1790785512.246002970,10046,10056 +1790785512.351506061,10046,10056 +1790785512.457397935,10046,10056 +1790785512.560686878,10062,10056 +1790785512.665251386,10046,10056 +1790785512.768095933,10046,10056 +1790785512.872327057,10046,10056 +1790785512.977400262,9742,10056 +1790785513.080707764,10046,10056 +1790785513.187673582,10046,10056 +1790785513.288432083,10046,10056 +1790785513.399750715,9678,10056 +1790785513.506683972,10046,10056 +1790785513.610784005,10046,10056 +1790785513.711428565,10046,10056 +1790785513.813815796,8814,8824 +1790785513.922006929,10046,10056 +1790785514.028088636,10046,10056 +1790785514.133812628,10046,10056 +1790785514.238145904,10046,10056 +1790785514.344378522,10046,10056 +1790785514.448952851,10046,10056 +1790785514.555555834,10046,10056 +1790785514.664039851,9718,10056 +1790785514.769987206,10046,10056 +1790785514.876405087,10046,10056 +1790785514.976876625,10046,10056 +1790785515.081801698,9654,10056 +1790785515.189514704,10046,10056 +1790785515.295486030,10046,10056 +1790785515.401983752,10046,10056 +1790785515.506359308,8814,9378 +1790785515.612007489,10046,10056 +1790785515.716437546,10046,10056 +1790785515.819026490,10046,10056 +1790785515.925976598,10014,10056 +1790785516.028326398,10046,10056 +1790785516.137486954,10046,10056 +1790785516.241794998,10046,10056 +1790785516.347212717,9254,10056 +1790785516.453439976,10046,10056 +1790785516.557944774,10046,10056 +1790785516.661908104,10046,10056 +1790785516.771654416,8820,10056 +1790785516.876828062,10046,10056 +1790785516.980235736,10046,10056 +1790785517.087214754,10046,10056 +1790785517.186735800,8814,8824 +1790785517.291730673,10046,10056 +1790785517.397234584,10046,10056 +1790785517.505629919,10046,10056 +1790785517.609422358,10046,8824 +1790785517.714923957,10046,10056 +1790785517.819185162,10046,10056 +1790785517.924483349,10046,10056 +1790785518.031988714,10046,8824 +1790785518.135062153,10046,10056 +1790785518.236837877,10046,10056 +1790785518.343327739,10046,10056 +1790785518.457928151,10046,9378 +1790785518.569565506,10046,10056 +1790785518.679258498,10046,10056 +1790785518.788909759,10046,10056 +1790785518.897217043,8814,8824 +1790785519.009419316,10046,10056 +1790785519.115791386,10046,10056 +1790785519.225636319,10046,10056 +1790785519.334354599,10014,10056 +1790785519.440040080,10046,10056 +1790785519.546072967,10046,10056 +1790785519.653674222,10046,10056 +1790785519.762148319,8822,10056 +1790785519.868606431,10046,10056 +1790785519.975218244,10046,10056 +1790785520.079465378,10046,10056 +1790785520.185106130,8814,10056 +1790785520.296538913,10046,10056 +1790785520.398585370,10046,10056 +1790785520.503000336,10046,10056 +1790785520.607729226,8814,10056 +1790785520.715561595,10046,10056 +1790785520.820370666,10046,10056 +1790785520.928698771,10046,10056 +1790785521.027942403,8814,10056 +1790785521.129727857,10046,10056 +1790785521.237391204,10046,10056 +1790785521.345066301,10046,10056 +1790785521.452451603,8814,10056 +1790785521.557364046,10046,10056 +1790785521.662049655,10046,10056 +1790785521.766428172,10046,10056 +1790785521.871871770,9276,10056 +1790785521.975600219,10046,10056 +1790785522.089898807,10046,10056 +1790785522.200866904,10046,10056 +1790785522.312099644,8814,10056 +1790785522.426897358,10046,10056 +1790785522.536036263,10046,10056 +1790785522.647051351,10046,10056 +1790785522.757902266,10046,10056 +1790785522.868640071,10046,10056 +1790785522.983191692,10046,10056 +1790785523.089651233,10046,10136 +1790785523.193531183,10046,10056 +1790785523.305394842,10046,10056 +1790785523.417698065,10046,10056 +1790785523.529687285,9974,10056 +1790785523.641811826,10046,10056 +1790785523.753596723,10046,10056 +1790785523.864895835,10046,10056 +1790785523.977742825,8814,10056 +1790785524.090148659,10046,10056 +1790785524.205878735,10046,10056 +1790785524.314467523,10046,10056 +1790785524.428434317,10046,10056 +1790785524.542309720,10046,10056 +1790785524.653150595,10046,10056 +1790785524.761193137,10046,8824 +1790785524.877135755,10046,10056 +1790785524.985373879,10046,10056 +1790785525.085483422,10046,10056 +1790785525.192940656,10046,9544 +1790785525.294966563,10046,10056 +1790785525.400312630,10046,10056 +1790785525.507198638,10046,10056 +1790785525.610071405,10046,8830 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/container.id b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/container.id new file mode 100644 index 0000000..2b4a2fe --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/container.id @@ -0,0 +1 @@ +297e8eb1cac5b35462bd8ab9c918791fb84c064372f3280596b1911dfb5c5f92 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/events.txt b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/events.txt new file mode 100644 index 0000000..6cf8ad7 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/events.txt @@ -0,0 +1,3 @@ +1790785597.522606520 sets:start +1790785680.424011199 sets:done +gpu3 after: 2 MiB diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/health-end.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/health-end.json new file mode 100644 index 0000000..7fb2695 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/health-end.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":8.082,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/health-rest.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/health-rest.json new file mode 100644 index 0000000..37aac99 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/health-rest.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.969,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/sets.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/sets.json new file mode 100644 index 0000000..7072da0 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/sets.json @@ -0,0 +1 @@ +{"label": "r2", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:26:37Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.44668398052454, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 38.12}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.255457205697894, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.58}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.109945038333535, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.55}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.041146075353026, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 35.06}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.23380698263645, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.46}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.057333927601576, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.49}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97197581641376, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.68}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30079997330904, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.72}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.89894701912999, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.77}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.564093014225364, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.73}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42346097715199, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.52}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.418731885030866, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 35.58}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4688720330596, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.58}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.6795301977545, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 41.77}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.102279184386134, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 35.2}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.841886980459094, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.94}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.684354931116104, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.74}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61470317840576, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.79}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23300900310278, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.43}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84897597506642, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.93}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.062718929722905, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 35.21}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.452701944857836, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.57}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.88074618205428, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 35.0}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92958799749613, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 35.05}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60902397334576, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.74}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.752744974568486, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.88}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.490472896024585, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.65}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66804398037493, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.76}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33147005550563, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.47}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82236692495644, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.0}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58252295665443, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.75}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63361391611397, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.76}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57617203518748, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.73}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.672403967007995, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.78}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25764015689492, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.4}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59895311668515, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.74}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.300289841368794, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.46}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31851000338793, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.45}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32272982597351, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.46}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48894203454256, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.57}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.293039029464126, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.37}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.315950030460954, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.47}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.234978983178735, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.38}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44298010133207, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.54}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45967103168368, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 34.49}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40479098632932, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.55}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13702689856291, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.33}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59655309841037, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.72}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.344638956710696, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.51}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47712215408683, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.52}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14432683773339, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.36}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56458195857704, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.72}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33831108547747, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.42}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60275291278958, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.7}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3884211294353, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.51}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54282207228243, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.64}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.461671978235245, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.58}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49944199807942, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.57}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22184803336859, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.41}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49661287292838, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.64}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26034890860319, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.37}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.241797894239426, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.4}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.064466858282685, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.24}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36752993427217, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.55}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14832803234458, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.32}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37176116555929, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.53}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15838794596493, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.31}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.709065014496446, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.84}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.140268133953214, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.28}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.402970949187875, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.6}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.136767990887165, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.31}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2613191138953, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.44}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.276949966326356, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.48}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.504761945456266, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.65}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.104306975379586, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.29}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.574122892692685, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.71}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.558474043384194, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 35.71}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75359503924847, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.79}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21709898672998, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.35}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82551595941186, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.82}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34837006777525, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.48}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49870206229389, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.61}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.938305012881756, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.08}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.715474143624306, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.92}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47463086619973, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.59}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.618294125422835, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.79}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23594802431762, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.4}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53924183547497, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.7}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22117794677615, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.41}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.705104099586606, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.84}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96622606366873, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.18}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.97163190878928, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 41.08}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.766474997624755, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.88}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.513872142881155, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.68}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.347269942983985, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.49}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79934602789581, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.89}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3603911139071, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.57}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74931514635682, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.89}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33156993985176, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.51}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.599793169647455, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.76}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59808302856982, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.75}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67648502252996, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.76}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66951407305896, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.76}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86172717809677, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 35.01}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.759405095130205, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.91}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35512890852988, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.41}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71241202391684, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 33.89}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59118605032563, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 35.98}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.277749041095376, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.66}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94523794203997, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 35.26}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.200227146968246, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.59}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48643305152655, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.79}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96565609425306, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.41}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40922105498612, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.8}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22486901097, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.47}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.399559980258346, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.78}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.018266040831804, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.39}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38428093492985, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.63}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.28939692862332, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.55}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.671971902251244, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.24}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11870708502829, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.16}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.286829901859164, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.25}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74763301387429, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 33.92}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.240659119561315, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.37}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94587596692145, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.05}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.066776072606444, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.27}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.676561925560236, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 33.86}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91032505594194, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.1}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.54568991623819, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 33.73}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03512600623071, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.2}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62411100231111, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 33.82}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22025919705629, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.39}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80411400273442, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 33.98}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87799395993352, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.02}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97245521470904, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.12}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04882599413395, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.17}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03352599404752, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.16}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04973589628935, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.24}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19552792608738, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.33}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20606816932559, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.25}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01919503509998, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.09}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23724908195436, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.29}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.040065973997116, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.01}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.175887962803245, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.18}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.032806780189276, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.02}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.464231019839644, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 34.52}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25660894997418, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.29}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28960920125246, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.26}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14469796977937, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.24}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.456660997122526, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 34.52}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.822943948209286, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 33.87}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4224609695375, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.49}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.151167169213295, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 34.38}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05082694068551, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.26}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.054846063256264, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.25}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24084887653589, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 34.53}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23278781399131, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.46}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949565168470144, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.17}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90587496384978, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 33.96}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16240799985826, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.46}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53792920522392, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 33.75}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05193684436381, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 34.19}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.715112160891294, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 33.91}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91852488555014, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 34.21}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.991525907069445, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 34.04}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03861706890166, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.17}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.492559127509594, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 33.66}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24692901596427, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.45}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01126598753035, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.18}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.041685776785016, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 34.23}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30337996967137, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 34.58}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.10791198723018, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 36.24}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17223787680268, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 34.34}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.467902060598135, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 34.74}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85023309662938, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 33.98}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.229899084195495, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.26}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9270951487124, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 34.06}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85475396923721, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 34.1}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03950601629913, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 34.26}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46401215903461, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 34.62}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.678232019767165, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 33.91}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82357398793101, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 34.06}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.560420179739594, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 33.81}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13636798784137, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.39}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65757109411061, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 33.97}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0757170021534, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 34.28}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.037295987829566, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.3}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.973054986447096, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 34.17}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12979797273874, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 34.28}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.153696874156594, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.44}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.977285984903574, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 34.18}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.103047128766775, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.36}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99741582386196, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 34.29}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.187947891652584, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 34.37}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05677599646151, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 34.35}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13394794613123, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.49}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.61223012022674, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 33.81}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61716396547854, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 34.83}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.766533041372895, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 34.0}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42686007171869, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 34.69}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.841783905401826, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.12}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.059046095237136, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.37}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78101394325495, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 34.04}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69521391950548, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 34.95}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62512102164328, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.75}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01238613575697, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.31}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.55640012584627, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 33.84}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.884255008772016, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 34.02}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67792188934982, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 33.82}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17468795366585, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 34.44}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.903555039316416, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.09}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8710239417851, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 34.1}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06657700240612, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 34.22}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.141327979043126, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 34.37}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59104090183973, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 33.85}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37713093869388, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 34.57}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9736950378865, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.22}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41724090650678, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 34.7}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39163000136614, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 34.5}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.178847938776016, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.4}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06358699128032, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 34.28}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.701164139434695, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 34.96}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09447709843516, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.3}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05421695485711, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.22}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.594703026115894, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 34.8}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.429180927574635, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.59}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.037425907328725, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 34.34}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6131241209805, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 34.91}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.063237046822906, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 34.32}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28917986154556, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.58}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.156777922064066, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 34.44}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.176007870584726, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.29}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79815400391817, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 34.01}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26775920949876, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 34.57}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48185892961919, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 33.75}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01904695481062, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 34.16}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83579410426319, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 34.07}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.314829882234335, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 34.52}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.620229948312044, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 33.82}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17690892517567, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 34.43}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.650391994044185, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 33.99}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17411798425019, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 34.38}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.692542023956776, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 33.89}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27822890505195, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.42}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.627341993153095, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 33.89}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.280819145962596, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.38}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75903403200209, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.02}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.4424160476774, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 35.39}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.247860968112946, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 34.4}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.975396098569036, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.1}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.820653825998306, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 34.09}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26641810499132, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.47}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.309879902750254, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 34.53}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11046697571874, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.147146882489324, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 34.32}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00666595250368, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.11}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46000095270574, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 34.64}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15690807439387, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.745243007317185, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 33.86}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.208799039945006, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.123706890270114, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 34.36}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83673404157162, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.07}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20814804546535, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 34.33}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9010039139539, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.09}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.494962103664875, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00039582327008, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.198618054389954, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96828614920378, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.26}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33603982254863, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07188693806529, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.31}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.789454916492105, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.024616030976176, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39601094275713, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 34.63}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01258497312665, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.34}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87881398946047, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 34.11}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47443202883005, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.67}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3000199906528, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98909608647227, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.24}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3727201949805, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.957825999706984, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184017011895776, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 34.44}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94300493225455, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.17}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27197996340692, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.226418033242226, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2073189355433, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 34.47}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.059335969388485, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.765356013551354, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 34.94}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10658801719546, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.83852490596473, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 35.14}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.205108907073736, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.422270979732275, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 35.11}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.08454414643347, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.34}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.91087204404175, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 34.31}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8973040021956, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.23}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1749890036881, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 34.45}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10568696260452, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41049114614725, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 34.63}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219329038634896, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.37}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42708987556398, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 34.7}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.258289055898786, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34290017746389, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 34.66}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69366193935275, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.91}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22388902679086, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.979005904868245, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.24}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1631879825145, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949384957551956, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.1}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32484998181462, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 34.61}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11949707940221, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21609804779291, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30309009552002, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2936490457505, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 34.62}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01664600521326, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.28}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17129793949425, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 34.5}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06913720816374, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.220368860289454, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 34.39}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.902404157444835, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.16}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11383696645498, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62745212204754, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.89}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67316392622888, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 34.81}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.784562980756164, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.91}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18891809694469, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.266249069944024, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.36}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27248916402459, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 34.41}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49627200886607, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.71}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81487403810024, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 33.95}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01546601951122, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184678949415684, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 34.27}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80344391614199, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03377698548138, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 34.28}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848853807896376, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.1}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.529348930343986, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 33.82}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56332101672888, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79147292859852, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79122403264046, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.0}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88425407744944, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84158390387893, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.1}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71621298417449, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 33.89}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.785762989893556, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.05}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92184598185122, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95285506546497, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.927014822140336, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848923889920115, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.0}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66127077303827, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95705602876842, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.18}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.627830006182194, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 33.91}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57245100475848, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.83}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90529405139387, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 34.2}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94750485010445, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.23}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70207201503217, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95761505514383, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.23}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84633388184011, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.04}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88773396424949, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69330198317766, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 33.94}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68989217840135, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75332199782133, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 33.94}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67185199446976, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.01}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.927883978933096, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.083216942846775, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.14}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219209130853415, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 34.42}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.193749099969864, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.4}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15861798264086, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 34.09}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02912596054375, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.12}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99467587284744, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.024846067652106, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07614680752158, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 34.28}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06888691522181, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.32}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06672601401806, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.308290135115385, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0520269498229, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 34.38}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15148800984025, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.27}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9738859590143, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31427900493145, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.52}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93277505226433, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 34.19}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.197297904640436, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99111589230597, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.249339977279305, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.57}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.377950036898255, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.064017029479146, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.33}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11127699352801, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 34.38}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.815223982557654, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.865063942968845, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.103266920894384, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.37}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15607803314924, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 34.37}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67656099423766, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.95}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0293369051069, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 34.3}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98464599438012, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.34}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15182701125741, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77299292758107, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97705501690507, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 34.25}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08778694272041, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.39}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91980498842895, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34738007187843, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.36}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.924261858686805, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 37.02}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95264412090182, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.27}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28411200083792, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 35.37}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12979797273874, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.628632970154285, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 34.75}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88197489827871, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.05}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.768535083159804, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32177009806037, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.57}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92222682386637, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 35.14}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91294486634433, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.99}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.95208805240691, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02805587090552, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.25}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74021496064961, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.197898142039776, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.49}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86863596178591, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 35.02}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97245614603162, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.17}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.092120084911585, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 35.33}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.118196953088045, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35970303043723, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 35.57}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19504806213081, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63027302920818, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13502804748714, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.22}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75556501932442, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33367007225752, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.64}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.09155896306038, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 35.38}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.550612119957805, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.67}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.97404794022441, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 35.1}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37356001511216, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.59}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52358304336667, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 34.7}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25181892514229, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.51}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.108499022200704, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 35.4}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26628888398409, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.54}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.988678086549044, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 35.24}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70997195877135, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.97}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.842626217752695, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 35.0}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85002401471138, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.06}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69075404852629, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71159199438989, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.01}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.581582786515355, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 34.78}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05278690904379, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.29}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.831834983080626, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 35.06}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1521080378443, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.37}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.99200803041458, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 35.29}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26338897831738, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.764016073197126, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 34.91}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10448695346713, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.39}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.12199993990362, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 35.24}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.142917048186064, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.44}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08706998825073, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 35.28}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92029579542577, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.26}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.770704831928015, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 34.95}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06533694639802, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.31}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72488492541015, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 34.95}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.109137047082186, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.32}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.10638911090791, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 35.29}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2432479634881, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.39}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01732882857323, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.2}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.212077993899584, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.80165477655828, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 34.9}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18247916549444, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.45}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79603494331241, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10946696810424, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.316902006044984, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 35.49}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.356629967689514, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.58}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7611662261188, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01378614641726, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.24}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.682224065065384, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00655605643988, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.33}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77555492706597, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.0}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73664294183254, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 33.99}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68605412915349, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 34.9}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.720452131703496, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 33.97}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.727355025708675, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 34.96}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89654404297471, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.06}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44949190877378, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 34.69}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88608482480049, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.09}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56884206086397, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 34.72}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.933585070073605, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.28}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 51.05949891731143, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 40.58}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.86027913540602, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.99}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.907259004190564, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 36.09}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.44921789318323, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 37.53}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.60481798276305, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 36.69}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61141304858029, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.50120515562594, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 35.66}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01093902438879, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.33}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.66063617169857, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 35.84}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.927808007225394, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.17}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.34659280255437, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 35.54}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64046402648091, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08962902985513, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33282000757754, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.6}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08367904275656, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 35.28}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31854003854096, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.52}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.00690802745521, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 35.21}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30955999344587, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.83918614499271, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.01}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.026925943791866, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.26}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.99337800592184, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 35.18}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.368229823186994, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.53}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13883988186717, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.34}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98440594598651, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.27}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90758713148534, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 35.13}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08416689187288, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.32}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75140503421426, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92890494130552, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.19}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.00300811417401, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 35.1}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69216204434633, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 33.89}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50622202642262, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 34.64}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03443589620292, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.15}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93478710576892, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.111018085852265, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.25}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08818897232413, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 35.18}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58095297962427, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.67}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.36012305691838, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 35.46}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34349994920194, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.42}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.27393217943609, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.37}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37106989137828, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.64}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.05428883060813, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.29}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.386061158031225, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.53}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23584015481174, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 35.34}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39719116128981, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79989620670676, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 34.99}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42543002590537, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.805835949257016, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 34.9}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22162893973291, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.47}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.091839058324695, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 35.24}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25028899498284, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.51}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.114198949187994, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 35.25}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.227059016004205, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.55}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98735816963017, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.13}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83857386931777, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.16}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90008695796132, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 35.13}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19667810946703, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.43}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.04708900861442, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 35.28}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.177417892962694, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.29}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98843780532479, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 35.11}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23614918999374, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.33}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.05192792788148, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 35.08}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26732009090483, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.36}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66590403392911, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 34.77}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19124780036509, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.25}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28226192854345, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 35.38}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55816305056214, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.68}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.928147008642554, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 34.93}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34753094613552, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.5}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.087488850578666, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38199989311397, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81934594735503, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 34.83}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10978711023927, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.39}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.88401698693633, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25325898267329, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.49}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7266148570925, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 34.98}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26948914118111, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.47}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.192749954760075, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15572785586119, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.32}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.883436074480414, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20280900597572, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.44}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73043514229357, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36678999848664, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.45}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76174504309893, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 34.99}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.017345894128084, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.23}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.609933177009225, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 34.71}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98468510992825, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.17}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36928990110755, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 34.53}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.135668098926544, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.38}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82323584891856, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.01}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1592880692333, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.3}], "seconds": 21.0, "failures": 0}, "rotations": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.92347499914467, "probs": [0.021278230729761302, 0.041549605758459535, 0.9371721635117792], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.96534493379295, "probs": [0.9569751557868662, 0.022187022115204392, 0.020837822097929372], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42147886008024, "probs": [0.028764030986406904, 0.9453276881980787, 0.025908280815514436], "top": "supported", "agreement": 1.0, "input_tokens": 196}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.69498199038208, "probs": [0.9253638152569406, 0.02190751448970375, 0.052728670253355744], "top": "insufficient", "agreement": 1.0, "input_tokens": 196}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39131797850132, "probs": [0.9597742641946788, 0.021791404944545737, 0.01843433086077556], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22496607154608, "probs": [0.0761485956363141, 0.025667636008847486, 0.8981837683548385], "top": "contradicted", "agreement": 1.0, "input_tokens": 206}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.49095994792879, "probs": [0.9159676813214028, 0.0555726559991739, 0.028459662679423257], "top": "contradicted", "agreement": 1.0, "input_tokens": 208}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65389203839004, "probs": [0.02148731809580675, 0.05171731055352475, 0.9267953713506685], "top": "insufficient", "agreement": 1.0, "input_tokens": 200}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65612185746431, "probs": [0.025695874936561842, 0.016912927133118746, 0.9573911979303193], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41266901418567, "probs": [0.9650011484904946, 0.014421131045601873, 0.02057772046390364], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.15835897624493, "probs": [0.9685134503822489, 0.011261347946432, 0.020225201671319158], "top": "supported", "agreement": 1.0, "input_tokens": 193}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39189912378788, "probs": [0.015601263068851802, 0.9601947660643307, 0.024203970866817515], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98396283946931, "probs": [0.008885594588144235, 0.029266695521859745, 0.9618477098899961], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22534605115652, "probs": [0.0467350628868348, 0.02343846611289458, 0.9298264710002706], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.68849183619022, "probs": [0.02661110718913202, 0.02251153411335893, 0.950877358697509], "top": "contradicted", "agreement": 1.0, "input_tokens": 199}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70519207976758, "probs": [0.9629993612353499, 0.018113485593456775, 0.01888715317119321], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24773597717285, "probs": [0.8820069055961203, 0.07635736188763995, 0.04163573251623969], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76509311795235, "probs": [0.02701681560472035, 0.9453954270569487, 0.027587757338330996], "top": "supported", "agreement": 1.0, "input_tokens": 198}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73087190277874, "probs": [0.02293683426823875, 0.9487912139983355, 0.028271951733425657], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70189310237765, "probs": [0.019367602097265263, 0.013573064959569787, 0.9670593329431649], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03357299603522, "probs": [0.03987925527749218, 0.02220468207692111, 0.9379160626455868], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.41814388893545, "probs": [0.02674495094579214, 0.9358820879397224, 0.03737296111448533], "top": "contradicted", "agreement": 1.0, "input_tokens": 211}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3368390314281, "probs": [0.03051840202788244, 0.9419931233693989, 0.027488474602718756], "top": "contradicted", "agreement": 1.0, "input_tokens": 201}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.7668428402394, "probs": [0.9573434428870992, 0.022664616232544885, 0.019991940880355972], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18864495679736, "probs": [0.20778523836492876, 0.6164398690397396, 0.17577489259533158], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.51035008393228, "probs": [0.01494019219967109, 0.958782768824845, 0.026277038975483776], "top": "permitted", "agreement": 1.0, "input_tokens": 218}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.67838209681213, "probs": [0.038717451842913995, 0.06953592628537188, 0.8917466218717142], "top": "permitted", "agreement": 1.0, "input_tokens": 203}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5801110342145, "probs": [0.8769652162510786, 0.06982844563770668, 0.053206338111214745], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18211591988802, "probs": [0.8731504724710493, 0.09124479732837286, 0.03560473020057783], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5503198876977, "probs": [0.03149783270455631, 0.9131032971636024, 0.055398870131841214], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21073616109788, "probs": [0.1973515606103041, 0.4555434753670417, 0.3471049640226543], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 209}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13833607546985, "probs": [0.02274552846853418, 0.9023369803119639, 0.07491749121950192], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13438587076962, "probs": [0.09302877372922651, 0.07088401170776323, 0.8360872145630103], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.49622913822532, "probs": [0.8635450750116728, 0.09024102515984218, 0.04621389982848493], "top": "permitted", "agreement": 1.0, "input_tokens": 211}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43088987469673, "probs": [0.028891977481104857, 0.0412263798327744, 0.9298816426861207], "top": "permitted", "agreement": 1.0, "input_tokens": 202}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8149931486696, "probs": [0.8174524855587436, 0.08365642576901129, 0.098891088672245], "top": "insufficient", "agreement": 1.0, "input_tokens": 204}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00847385264933, "probs": [0.017660819839882137, 0.04340570512192549, 0.9389334750381924], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32240709289908, "probs": [0.06132539226815876, 0.8373735739130129, 0.10130103381882835], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.02946400083601, "probs": [0.887606329608409, 0.06778068214997746, 0.04461298824161357], "top": "prohibited", "agreement": 1.0, "input_tokens": 213}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16801499761641, "probs": [0.6913075067568434, 0.07074700515018907, 0.23794548809296753], "top": "insufficient", "agreement": 1.0, "input_tokens": 212}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3632078692317, "probs": [0.9055821252398959, 0.059736224410951576, 0.03468165034915258], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54104996845126, "probs": [0.034796330955504114, 0.9473839012637663, 0.017819767780729617], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00907409004867, "probs": [0.06676731835885638, 0.04041944853804973, 0.8928132331030938], "top": "permitted", "agreement": 1.0, "input_tokens": 208}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38725787773728, "probs": [0.0792938381036179, 0.09571489465194817, 0.8249912672444338], "top": "insufficient", "agreement": 1.0, "input_tokens": 215}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19659682177007, "probs": [0.08689479609139293, 0.6606481239315956, 0.2524570799770115], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8967539574951, "probs": [0.8171805101277473, 0.07074519344147624, 0.11207429643077639], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.55293108336627, "probs": [0.103178626971303, 0.7367489124270009, 0.16007246060169616], "top": "prohibited", "agreement": 1.0, "input_tokens": 209}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2475279904902, "probs": [0.08560085550345033, 0.3403340342335013, 0.5740651102630484], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.52908287011087, "probs": [0.017814439341391167, 0.9671155236327101, 0.015070037025898734], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.42857197672129, "probs": [0.9513649777837366, 0.01716163808274369, 0.03147338413351958], "top": "B", "agreement": 1.0, "input_tokens": 201}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.16081696934998, "probs": [0.025903351803450637, 0.9255875837884197, 0.048509064408129646], "top": "B", "agreement": 1.0, "input_tokens": 192}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.50336206890643, "probs": [0.019189428631759722, 0.9189139403005832, 0.061896631067657096], "top": "A", "agreement": 1.0, "input_tokens": 192}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02719499729574, "probs": [0.016185500037668298, 0.9553471934068705, 0.028467306555461265], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.10950808040798, "probs": [0.014798671264711157, 0.9697705732073082, 0.015430755527980513], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19649803265929, "probs": [0.025560818133635414, 0.022080003391839246, 0.9523591784745253], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08377703465521, "probs": [0.05311118840492001, 0.6811115463281155, 0.2657772652669644], "top": "A", "agreement": 1.0, "input_tokens": 190}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.83602606877685, "probs": [0.026144179410345973, 0.9539350561208116, 0.019920764468842493], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.79188400879502, "probs": [0.9592258084083156, 0.024179546083237823, 0.016594645508446512], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.38954117894173, "probs": [0.015353470774838725, 0.9649135180343748, 0.019733011190786628], "top": "B", "agreement": 1.0, "input_tokens": 197}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.29839007742703, "probs": [0.6229250573303873, 0.29341037282098503, 0.08366456984862763], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43460003100336, "probs": [0.9523442934756741, 0.028977529107373938, 0.018678177416952112], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.87978409603238, "probs": [0.9509486773226812, 0.026062332698517814, 0.022988989978801007], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11371586471796, "probs": [0.028605829867980576, 0.9599959678670565, 0.01139820226496276], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.58347101323307, "probs": [0.027524206942156276, 0.10494883289705875, 0.8675269601607849], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08665489032865, "probs": [0.025763001405599038, 0.9598922160922182, 0.014344782502182778], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93993519432843, "probs": [0.01775221293583573, 0.018510449440082843, 0.9637373376240814], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2998401466757, "probs": [0.05805098351023624, 0.04331699912016453, 0.8986320173695993], "top": "B", "agreement": 1.0, "input_tokens": 200}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.24862788245082, "probs": [0.40713917224854596, 0.04723599385140831, 0.5456248339000457], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1625680886209, "probs": [0.9628824551152317, 0.020107611199832023, 0.017009933684936158], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90692493133247, "probs": [0.017791047056264133, 0.965845589126288, 0.016363363817447876], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.35855002328753, "probs": [0.9697536223339248, 0.01200584747578968, 0.018240530190285528], "top": "A", "agreement": 1.0, "input_tokens": 194}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56981309875846, "probs": [0.46166613268095874, 0.4246186674610029, 0.11371519985803841], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 192}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63864389061928, "probs": [0.9434733126902809, 0.03465269924248234, 0.021873988067236654], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.29545990377665, "probs": [0.05653147014528852, 0.049865123655206005, 0.8936034061995054], "top": "supported", "agreement": 1.0, "input_tokens": 214}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1862378846854, "probs": [0.9646005420197569, 0.019726610085808626, 0.01567284789443457], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70744192227721, "probs": [0.011200303179937986, 0.9433281975088483, 0.045471499311213706], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3328399322927, "probs": [0.04435070832583449, 0.8823880770841831, 0.07326121458998254], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22806893661618, "probs": [0.38436219601806726, 0.39248484865749256, 0.22315295532444018], "top": "prohibited", "agreement": 0.3333333333333333, "input_tokens": 229}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.13498720340431, "probs": [0.038671196720481874, 0.051824945670009545, 0.9095038576095087], "top": "permitted", "agreement": 1.0, "input_tokens": 222}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56370292417705, "probs": [0.9386225473366789, 0.024670771221418437, 0.03670668144190261], "top": "insufficient", "agreement": 1.0, "input_tokens": 221}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2614789698273, "probs": [0.06338502047709155, 0.20445210704639485, 0.7321628724765136], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1336169950664, "probs": [0.01023906219590939, 0.9776572412337166, 0.012103696570374124], "top": "A", "agreement": 1.0, "input_tokens": 210}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22118902392685, "probs": [0.08343895995982653, 0.048442981731284813, 0.8681180583088886], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.68476212210953, "probs": [0.028590681320234453, 0.07026846391461965, 0.9011408547651459], "top": "insufficient", "agreement": 1.0, "input_tokens": 211}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27347999252379, "probs": [0.9485370017696566, 0.021991387483579136, 0.02947161074676416], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57483316026628, "probs": [0.1476984273576982, 0.06533845399896182, 0.78696311864334], "top": "supported", "agreement": 1.0, "input_tokens": 227}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.47177207283676, "probs": [0.9659641119707486, 0.01300233393992522, 0.021033554089326202], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55744280666113, "probs": [0.9452042665175415, 0.037745228086007734, 0.01705050539645064], "top": "insufficient", "agreement": 1.0, "input_tokens": 202}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.91050493530929, "probs": [0.7928775526060625, 0.1207272484983066, 0.08639519889563085], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27028997987509, "probs": [0.4902958520500278, 0.23093932363732236, 0.2787648243126498], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 231}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11481708474457, "probs": [0.08132590401657058, 0.05465960600149562, 0.8640144899819338], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22538882307708, "probs": [0.2290171921944468, 0.6937872957432885, 0.07719551206226469], "top": "insufficient", "agreement": 1.0, "input_tokens": 223}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.17888811975718, "probs": [0.030141465776599607, 0.9500195362761745, 0.019838997947225906], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90977501124144, "probs": [0.025305433525381633, 0.03185065213808263, 0.9428439143365358], "top": "B", "agreement": 1.0, "input_tokens": 222}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.25929781235754, "probs": [0.05059101952456338, 0.04279722164027021, 0.9066117588351665], "top": "B", "agreement": 1.0, "input_tokens": 215}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.13965801894665, "probs": [0.012768350746295443, 0.01860437029559649, 0.9686272789581081], "top": "insufficient", "agreement": 1.0, "input_tokens": 201}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.95979611389339, "probs": [0.9208322432392101, 0.04082517423414474, 0.038342582526645214], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90040381066501, "probs": [0.8434852492825246, 0.05011598360172626, 0.10639876711574907], "top": "supported", "agreement": 1.0, "input_tokens": 219}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73847289197147, "probs": [0.023619578875477214, 0.01956735197412583, 0.9568130691503969], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27584701776505, "probs": [0.8650697558641954, 0.10248576421341996, 0.03244447992238454], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.56231089867651, "probs": [0.10549476831325036, 0.0754944842886127, 0.819010747398137], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38530792109668, "probs": [0.335350091795104, 0.11069809870287566, 0.5539518095020204], "top": "prohibited", "agreement": 1.0, "input_tokens": 221}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3536687977612, "probs": [0.8825022468959398, 0.07026933060840494, 0.047228422495655314], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01221404410899, "probs": [0.06932232571305925, 0.9077945737563922, 0.022883100530548473], "top": "insufficient", "agreement": 1.0, "input_tokens": 220}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.72152305394411, "probs": [0.009281733826954271, 0.9839356748468795, 0.006782591326166279], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18155503086746, "probs": [0.008340312669325892, 0.9815921692199244, 0.010067518110749797], "top": "B", "agreement": 1.0, "input_tokens": 230}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.5738529432565, "probs": [0.012023447059675983, 0.9711751830662149, 0.016801369874109195], "top": "B", "agreement": 1.0, "input_tokens": 223}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.15951504930854, "probs": [0.011643930956106696, 0.9806920781093156, 0.007663990934577689], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76964581944048, "probs": [0.900478485395825, 0.023178368052708644, 0.07634314655146633], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.36706796847284, "probs": [0.06499692312355161, 0.047496249800495424, 0.887506827075953], "top": "supported", "agreement": 1.0, "input_tokens": 224}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.81810399889946, "probs": [0.3922836361905562, 0.48352898160684454, 0.12418738220259931], "top": "insufficient", "agreement": 0.3333333333333333, "input_tokens": 220}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90648395009339, "probs": [0.0560499070635644, 0.020116182060737196, 0.9238339108756985], "top": "insufficient", "agreement": 1.0, "input_tokens": 208}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.863933917135, "probs": [0.17500642472273137, 0.21124871005668264, 0.613744865220586], "top": "permitted", "agreement": 1.0, "input_tokens": 227}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.85776709578931, "probs": [0.5914884424684399, 0.035143500631624254, 0.37336805689993585], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 228}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.78900296241045, "probs": [0.9046569284481161, 0.042704941598692904, 0.05263812995319102], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13665620237589, "probs": [0.029736313438628644, 0.033014009977779125, 0.9372496765835922], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.82867288030684, "probs": [0.7432028429041843, 0.03582480375679601, 0.2209723533390198], "top": "A", "agreement": 1.0, "input_tokens": 221}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19941593520343, "probs": [0.06632110060786917, 0.8329169964218126, 0.10076190297031828], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.72478221729398, "probs": [0.7397871533568318, 0.1860713523015022, 0.07414149434166593], "top": "A", "agreement": 1.0, "input_tokens": 207}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02457611821592, "probs": [0.9248773731002589, 0.014113602375072627, 0.06100902452466855], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.99382594227791, "probs": [0.9556486965100951, 0.021248690458660385, 0.02310261303124464], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.0531169641763, "probs": [0.010853780626859795, 0.015814708468139885, 0.9733315109050004], "top": "supported", "agreement": 1.0, "input_tokens": 212}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.62550090253353, "probs": [0.9606317563420671, 0.018450800471885003, 0.020917443186047876], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27171001397073, "probs": [0.8027819128710822, 0.011505476315571077, 0.18571261081334672], "top": "contradicted", "agreement": 0.6666666666666666, "input_tokens": 203}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.97806291095912, "probs": [0.8487397319059693, 0.05482797840153974, 0.09643228969249089], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.17209901101887, "probs": [0.019700876806489794, 0.943405397557902, 0.036893725635608235], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07685504853725, "probs": [0.9481975719388965, 0.021083015776241384, 0.030719412284862045], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24921701289713, "probs": [0.11594495476456004, 0.2790651305948641, 0.6049899146405758], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.80278101749718, "probs": [0.011620895003629282, 0.009627189199890869, 0.9787519157964798], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03300419077277, "probs": [0.9712341256474011, 0.015134214927072205, 0.013631659425526738], "top": "B", "agreement": 1.0, "input_tokens": 212}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17468606121838, "probs": [0.9748621568921099, 0.014266993806726613, 0.010870849301163494], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25843594223261, "probs": [0.01218702517662872, 0.9640155969735945, 0.02379737784977667], "top": "insufficient", "agreement": 1.0, "input_tokens": 191}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07350508123636, "probs": [0.05216159441960384, 0.25033305566222464, 0.6975053499181716], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98382314108312, "probs": [0.0744428867114499, 0.8598927356841719, 0.06566437760437807], "top": "supported", "agreement": 1.0, "input_tokens": 221}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9482929520309, "probs": [0.031673836031138235, 0.010902017412968387, 0.9574241465558934], "top": "supported", "agreement": 1.0, "input_tokens": 229}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21386587060988, "probs": [0.9598638711537073, 0.029206330829344665, 0.01092979801694802], "top": "insufficient", "agreement": 1.0, "input_tokens": 226}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12577602639794, "probs": [0.3126088342455135, 0.527299092068065, 0.1600920736864215], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11021502315998, "probs": [0.018596715235427216, 0.8540525288330012, 0.1273507559315715], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.35944905132055, "probs": [0.2682095319185024, 0.5694223884119577, 0.16236807966953995], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.05736386403441, "probs": [0.5197715159798874, 0.040530340274099394, 0.4396981437460133], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 224}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.28141702525318, "probs": [0.03677949530401433, 0.9404844838908847, 0.02273602080510092], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.19254400767386, "probs": [0.09722552658518739, 0.7089119171844285, 0.19386255623038412], "top": "B", "agreement": 1.0, "input_tokens": 218}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.53492093458772, "probs": [0.0446355264117619, 0.029378934066784906, 0.9259855395214532], "top": "B", "agreement": 1.0, "input_tokens": 209}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.34934792667627, "probs": [0.8956127457317613, 0.023540301062398936, 0.08084695320583983], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8773051481694, "probs": [0.9215420659013949, 0.05709241087032677, 0.021365523228278453], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38078797981143, "probs": [0.024650674719169527, 0.05690076079826144, 0.9184485644825692], "top": "contradicted", "agreement": 1.0, "input_tokens": 215}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1340859849006, "probs": [0.017881542110937673, 0.0314503320010902, 0.9506681258879721], "top": "contradicted", "agreement": 1.0, "input_tokens": 236}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84610195271671, "probs": [0.015971281239954956, 0.021403804942628207, 0.9626249138174169], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01892399042845, "probs": [0.9469841068616602, 0.027062216114961006, 0.025953677023378714], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.79540098458529, "probs": [0.9655262695437641, 0.017417092730978554, 0.01705663772525738], "top": "supported", "agreement": 1.0, "input_tokens": 241}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00976420007646, "probs": [0.9523229876734454, 0.019887100792948465, 0.02778991153360623], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9653619993478, "probs": [0.02944395390697739, 0.022909170093210977, 0.9476468759998116], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1517860032618, "probs": [0.028467305657075486, 0.01618550022217239, 0.9553471941207522], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.62260898202658, "probs": [0.9626180927581529, 0.027508993151357353, 0.009872914090489732], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.85486197285354, "probs": [0.01216177373585067, 0.025820060282101747, 0.9620181659820476], "top": "supported", "agreement": 1.0, "input_tokens": 216}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.75384001620114, "probs": [0.008947931903485177, 0.02245643770922697, 0.9685956303872878], "top": "supported", "agreement": 1.0, "input_tokens": 237}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21442606113851, "probs": [0.03490400868942244, 0.07256938070358598, 0.8925266106069916], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30745703727007, "probs": [0.8560321676779682, 0.10355825471147521, 0.04040957761055657], "top": "contradicted", "agreement": 1.0, "input_tokens": 209}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01811397261918, "probs": [0.9275028632254695, 0.04760344873379892, 0.024893688040731596], "top": "contradicted", "agreement": 1.0, "input_tokens": 230}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9652430228889, "probs": [0.9261949727483907, 0.021028996331540862, 0.0527760309200684], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06168496794999, "probs": [0.03674381458625419, 0.02368412668428037, 0.9395720587294654], "top": "supported", "agreement": 1.0, "input_tokens": 222}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12497485615313, "probs": [0.026670715956627202, 0.020321964093249996, 0.9530073199501229], "top": "supported", "agreement": 1.0, "input_tokens": 243}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76580301858485, "probs": [0.08330838841015804, 0.7031959025468014, 0.21349570904304058], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.48308002762496, "probs": [0.18172827933132565, 0.6645395854888777, 0.15373213517979664], "top": "prohibited", "agreement": 1.0, "input_tokens": 223}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.40966899134219, "probs": [0.08895150060315864, 0.816337598890086, 0.09471090050675535], "top": "prohibited", "agreement": 1.0, "input_tokens": 244}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.29607720673084, "probs": [0.0293692258945804, 0.11923451516955454, 0.851396258935865], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4426790215075, "probs": [0.7559638022373625, 0.22951644325448917, 0.014519754508148302], "top": "permitted", "agreement": 1.0, "input_tokens": 220}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54285999387503, "probs": [0.9180643670041458, 0.055699863652112935, 0.026235769343741142], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.88290495797992, "probs": [0.8545877320672847, 0.07877388866940015, 0.06663837926331509], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.88860395364463, "probs": [0.10813821274832802, 0.06970313933726799, 0.822158647914404], "top": "prohibited", "agreement": 1.0, "input_tokens": 219}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.23563919216394, "probs": [0.04460783183966404, 0.029981179056752127, 0.9254109891035839], "top": "prohibited", "agreement": 1.0, "input_tokens": 240}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.89105496183038, "probs": [0.9383472668786358, 0.03989758917731366, 0.021755143944050585], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42865889146924, "probs": [0.019454336721104887, 0.06822606373845527, 0.9123195995404397], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75674404390156, "probs": [0.015551699476841353, 0.0471124973010445, 0.9373358032221141], "top": "permitted", "agreement": 1.0, "input_tokens": 246}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4506590589881, "probs": [0.0377743579001357, 0.05503984914430352, 0.9071857929555608], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.15061805583537, "probs": [0.9057645436591656, 0.057300812015498664, 0.03693464432533565], "top": "prohibited", "agreement": 1.0, "input_tokens": 227}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30558787286282, "probs": [0.9388430475426657, 0.037491202062820105, 0.023665750394514205], "top": "prohibited", "agreement": 1.0, "input_tokens": 248}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08859506808221, "probs": [0.28824200219090185, 0.5868851532425197, 0.12487284456657842], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3235069848597, "probs": [0.5631497490596038, 0.19793031332732086, 0.2389199376130753], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22011916711926, "probs": [0.07442810557243966, 0.7426469601625133, 0.18292493426504697], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.87431693077087, "probs": [0.021424613092898774, 0.963560742046841, 0.01501464486026032], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57446295954287, "probs": [0.01601169462902707, 0.9650607227828113, 0.01892758258816167], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.99670582450926, "probs": [0.01772358905716697, 0.9621833982502599, 0.020093012692573263], "top": "A", "agreement": 1.0, "input_tokens": 232}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.5688740927726, "probs": [0.038484166247206124, 0.9437641516667067, 0.017751682086087198], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.420429026708, "probs": [0.01638885815779191, 0.9473306235405403, 0.036280518301667765], "top": "B", "agreement": 1.0, "input_tokens": 219}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14907585456967, "probs": [0.01830968230910091, 0.9532844759985375, 0.028405841692361555], "top": "B", "agreement": 1.0, "input_tokens": 240}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16815586015582, "probs": [0.023557953254122476, 0.9543166602232142, 0.022125386522663444], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.96405279263854, "probs": [0.031413960782038314, 0.9495687268473656, 0.01901731237059622], "top": "A", "agreement": 1.0, "input_tokens": 214}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27796717360616, "probs": [0.07602225290100428, 0.8966935357050799, 0.027284211393915826], "top": "A", "agreement": 1.0, "input_tokens": 235}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73298204690218, "probs": [0.023803230827026532, 0.0318997393139744, 0.944297029858999], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98225293122232, "probs": [0.9537738809196197, 0.029021027054749942, 0.017205092025630454], "top": "B", "agreement": 1.0, "input_tokens": 214}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22996610961854, "probs": [0.964600540916227, 0.019726610712202018, 0.015672848371570857], "top": "B", "agreement": 1.0, "input_tokens": 235}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1174949388951, "probs": [0.01331158113941839, 0.9684719956116256, 0.018216423248955952], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4553990252316, "probs": [0.03555637420973125, 0.9480424533538477, 0.016401172436420974], "top": "A", "agreement": 1.0, "input_tokens": 219}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30183697119355, "probs": [0.02192392382604651, 0.9656109109209706, 0.012465165252982892], "top": "A", "agreement": 1.0, "input_tokens": 240}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32300709746778, "probs": [0.019595497666027976, 0.022215172638678914, 0.9581893296952932], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24225700646639, "probs": [0.9596984415322811, 0.02225016024721796, 0.01805139822050088], "top": "B", "agreement": 1.0, "input_tokens": 217}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.0189440138638, "probs": [0.9679216672147071, 0.013872260676720507, 0.01820607210857243], "top": "B", "agreement": 1.0, "input_tokens": 238}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14141502790153, "probs": [0.02350526733620643, 0.04401818593722302, 0.9324765467265705], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22078583016992, "probs": [0.9307059696632914, 0.042134929049560656, 0.027159101287147974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16412579454482, "probs": [0.9518944397002193, 0.030200951503065683, 0.01790460879671502], "top": "contradicted", "agreement": 1.0, "input_tokens": 248}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.93329190649092, "probs": [0.0734946189438627, 0.8668797160257922, 0.05962566503034527], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13516492210329, "probs": [0.06641491931728377, 0.8517220237244112, 0.08186305695830509], "top": "prohibited", "agreement": 1.0, "input_tokens": 239}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.95432708598673, "probs": [0.026075603104276424, 0.9317425408247122, 0.04218185607101137], "top": "prohibited", "agreement": 1.0, "input_tokens": 260}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.91602511703968, "probs": [0.8272206536224163, 0.11345127657827278, 0.05932806979931101], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.37690808996558, "probs": [0.10398356939461242, 0.36466878809364145, 0.5313476425117462], "top": "A", "agreement": 0.3333333333333333, "input_tokens": 227}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24137713946402, "probs": [0.047442131153065406, 0.31816121638654793, 0.6343966524603867], "top": "A", "agreement": 1.0, "input_tokens": 248}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25071597658098, "probs": [0.026110501601006305, 0.021183264898242012, 0.9527062335007518], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.2236258983612, "probs": [0.9405169120595193, 0.024208959715331872, 0.035274128225148874], "top": "contradicted", "agreement": 1.0, "input_tokens": 237}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8280139695853, "probs": [0.9538802773453129, 0.023059861528293313, 0.023059861126393875], "top": "contradicted", "agreement": 1.0, "input_tokens": 258}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92808185517788, "probs": [0.11358428062013931, 0.09215012355248668, 0.7942655958273741], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84474105760455, "probs": [0.6104725669434667, 0.2700592902850353, 0.11946814277149793], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 244}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70086282677948, "probs": [0.8448473542913075, 0.08645996139090424, 0.06869268431778815], "top": "prohibited", "agreement": 1.0, "input_tokens": 265}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98788394033909, "probs": [0.0265024583113502, 0.9469950831682415, 0.02650245852040832], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03404401242733, "probs": [0.04213492809402675, 0.9307059697792428, 0.027159102126730544], "top": "A", "agreement": 1.0, "input_tokens": 233}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.33639811910689, "probs": [0.02140380544189133, 0.962624913264518, 0.01597128129359056], "top": "A", "agreement": 1.0, "input_tokens": 254}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84539205208421, "probs": [0.04515777084298974, 0.03741040508103143, 0.9174318240759788], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.20929587073624, "probs": [0.9100932546372905, 0.041201749446211355, 0.048704995916498045], "top": "contradicted", "agreement": 1.0, "input_tokens": 234}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.28158699162304, "probs": [0.9366656772497187, 0.038194709842211357, 0.025139612908070024], "top": "contradicted", "agreement": 1.0, "input_tokens": 255}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16747599467635, "probs": [0.868308675840529, 0.07676005878826689, 0.0549312653712041], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18059507012367, "probs": [0.15789450752928103, 0.11538080349900755, 0.7267246889717114], "top": "prohibited", "agreement": 1.0, "input_tokens": 235}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41363898664713, "probs": [0.08083885614456791, 0.06032105358305889, 0.8588400902723732], "top": "prohibited", "agreement": 1.0, "input_tokens": 256}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6059910915792, "probs": [0.009236476474052145, 0.9791380440055677, 0.011625479520380164], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.8796430695802, "probs": [0.014594034131288612, 0.9765710625943955, 0.008834903274315823], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90185411274433, "probs": [0.0070908823458139675, 0.986522232315347, 0.006386885338839195], "top": "A", "agreement": 1.0, "input_tokens": 259}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.1369068492204, "probs": [0.07423043782078662, 0.012813700792315847, 0.9129558613868974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.3259110134095, "probs": [0.9080827305344887, 0.025413463174314043, 0.06650380629119718], "top": "contradicted", "agreement": 1.0, "input_tokens": 238}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.56280708685517, "probs": [0.912724042631006, 0.024497031790175153, 0.06277892557881881], "top": "contradicted", "agreement": 1.0, "input_tokens": 259}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.38648287951946, "probs": [0.44029144656423663, 0.4049592579435585, 0.15474929549220479], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 227}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19482793845236, "probs": [0.36520001887064335, 0.21202778871080402, 0.4227721924185527], "top": "permitted", "agreement": 0.3333333333333333, "input_tokens": 238}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.80860606953502, "probs": [0.0870101345530018, 0.5040542344698868, 0.4089356309771113], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 259}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39454803802073, "probs": [0.4770731173522989, 0.04585376828987368, 0.4770731143578275], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 221}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.95601291768253, "probs": [0.39895641057043185, 0.0317669442524896, 0.5692766451770785], "top": "B", "agreement": 1.0, "input_tokens": 232}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11420504190028, "probs": [0.3779951490486449, 0.03557902847506642, 0.5864258224762886], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 253}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.99528306536376, "probs": [0.02269220266989126, 0.01879907857817761, 0.9585087187519311], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09157506749034, "probs": [0.9501164329419498, 0.021572126471092922, 0.028311440586957334], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.95081287808716, "probs": [0.9561058300755273, 0.022635315404623546, 0.021258854519849178], "top": "contradicted", "agreement": 1.0, "input_tokens": 245}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17760599032044, "probs": [0.09448034891197682, 0.0384419614966505, 0.8670776895913728], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03115295432508, "probs": [0.7989049560014004, 0.057297274549302536, 0.143797769449297], "top": "prohibited", "agreement": 1.0, "input_tokens": 229}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31923803500831, "probs": [0.9033714574378554, 0.03317980421079573, 0.06344873835134875], "top": "prohibited", "agreement": 1.0, "input_tokens": 250}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.89074210636318, "probs": [0.9807799827157122, 0.008509520309325268, 0.010710496974962463], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.26243690401316, "probs": [0.009842123709939055, 0.01026250243418525, 0.9798953738558757], "top": "A", "agreement": 1.0, "input_tokens": 217}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.82304303534329, "probs": [0.011627702527445261, 0.00904705287357599, 0.9793252445989787], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.35805812105536, "probs": [0.6618883595686608, 0.28081044656836285, 0.05730119386297648], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16671487130225, "probs": [0.06488320495606428, 0.37587178359700457, 0.5592450114469311], "top": "contradicted", "agreement": 1.0, "input_tokens": 235}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12407496571541, "probs": [0.06100397866886853, 0.1887116244179442, 0.7502843969131873], "top": "contradicted", "agreement": 1.0, "input_tokens": 256}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24153592996299, "probs": [0.17125015092489698, 0.5080506282282206, 0.32069922084688257], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60784093104303, "probs": [0.1514529519738628, 0.627869626193126, 0.22067742183301117], "top": "permitted", "agreement": 1.0, "input_tokens": 237}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.07885709032416, "probs": [0.17013095481276255, 0.7202069629772057, 0.10966208221003176], "top": "permitted", "agreement": 1.0, "input_tokens": 258}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49077291600406, "probs": [0.023818955965634256, 0.9449208465220887, 0.031260197512277065], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1828758791089, "probs": [0.04244531776287583, 0.9375620734741275, 0.01999260876299673], "top": "A", "agreement": 1.0, "input_tokens": 224}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.86997400969267, "probs": [0.028930382730149296, 0.9507948329241313, 0.02027478434571944], "top": "A", "agreement": 1.0, "input_tokens": 245}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86626503244042, "probs": [0.227340819830238, 0.7726591801697621], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18090707808733, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06262787804008, "probs": [0.16800751383797227, 0.8319924861620277], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.33539905771613, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86905620619655, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.39100879430771, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.32165902294219, "probs": [0.20097170058635486, 0.7990282994136452], "top": "ordinary", "agreement": 1.0, "input_tokens": 254}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06777785904706, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0551279373467, "probs": [0.13950622912843208, 0.8604937708715679], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4328798931092, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86461519449949, "probs": [0.3067002008775075, 0.6932997991224925], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.229426888749, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.944085881114, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "agreement": 1.0, "input_tokens": 256}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25466805882752, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99205690994859, "probs": [0.07740461434958126, 0.9225953856504188], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11000595614314, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61531201563776, "probs": [0.09196032674706488, 0.9080396732529351], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.08513591811061, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.82638510130346, "probs": [0.1552608265624769, 0.844739173437523], "top": "ordinary", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.23083784244955, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64636207744479, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.16289692930877, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.45620998926461, "probs": [0.32019975839988934, 0.6798002416001107], "top": "ordinary", "agreement": 1.0, "input_tokens": 255}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18803611956537, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67256297543645, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.53207086957991, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62682199850678, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.21526706032455, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66413310915232, "probs": [0.13578298208575287, 0.8642170179142472], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.13187597133219, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.57731195911765, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22672791592777, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71707616746426, "probs": [0.9710643563300382, 0.02893564366996184], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.29292818717659, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.25169008597732, "probs": [0.9673224283590844, 0.032677571640915566], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.26700808294117, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99369790032506, "probs": [0.9350120677910418, 0.06498793220895807], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27948803827167, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90570584498346, "probs": [0.8405803537312423, 0.1594196462687578], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.54220109805465, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1125090494752, "probs": [0.9290542127600319, 0.07094578723996812], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.06852601282299, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.513542836532, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.00813393481076, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72066299244761, "probs": [0.9516645319845058, 0.048335468015494064], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27767917700112, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03118689358234, "probs": [0.9439168350492196, 0.056083164950780265], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72017288394272, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43481099046767, "probs": [0.905386513027929, 0.09461348697207088], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.94557599723339, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.51935405656695, "probs": [0.9156019647894573, 0.0843980352105427], "top": "react", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4836809784174, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31442094780505, "probs": [0.6155100736384166, 0.38448992636158347], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.24630983546376, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.39514193497598, "probs": [0.6932998086882517, 0.30670019131174836], "top": "react", "agreement": 1.0, "input_tokens": 268}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.96631701663136, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.20541010610759, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.91460602916777, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.93324917554855, "probs": [0.915601962671152, 0.08439803732884785], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.5416120365262, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43232289329171, "probs": [0.40698895046193323, 0.5930110495380667], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06524815410376, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.95294601842761, "probs": [0.36247441705939704, 0.637525582940603], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85330521315336, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.09998811595142, "probs": [0.908039679856927, 0.09196032014307312], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.87348511070013, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.45882297866046, "probs": [0.3481052260909569, 0.6518947739090432], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28280091658235, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90084620378911, "probs": [0.5235093404303358, 0.4764906595696642], "top": "react", "agreement": 0.5, "input_tokens": 255}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8387661036104, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 76.22286002151668, "probs": [0.2743951410804348, 0.7256048589195652], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.47663289308548, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.42274011857808, "probs": [0.2112354509623397, 0.7887645490376604], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.75354403443635, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.21674011088908, "probs": [0.12511350715617336, 0.8748864928438266], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.69518410228193, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.81886490434408, "probs": [0.27439514286499256, 0.7256048571350073], "top": "ordinary", "agreement": 1.0, "input_tokens": 181}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63280202075839, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9403859693557, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76996488124132, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66156289167702, "probs": [0.4222145135481339, 0.5777854864518661], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77340402267873, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.55294204130769, "probs": [0.1472092305510008, 0.8527907694489993], "top": "ordinary", "agreement": 1.0, "input_tokens": 183}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63825305178761, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84939505532384, "probs": [0.11201237833636817, 0.8879876216636319], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28051009587944, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.45914086140692, "probs": [0.12172004831180805, 0.878279951688192], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.82387316040695, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.44548201560974, "probs": [0.17243807104776526, 0.8275619289522348], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30926800891757, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.81735406629741, "probs": [0.16366828025224447, 0.8363317197477556], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19656021334231, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.35790090635419, "probs": [0.3000708042310285, 0.6999291957689716], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76091405563056, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89006498083472, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.78742485120893, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.237340034917, "probs": [0.10592235643160026, 0.8940776435683997], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62408297881484, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79205399006605, "probs": [0.19108583142179145, 0.8089141685782085], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16640911065042, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4707300066948, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38593913987279, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.68244291096926, "probs": [0.7614507841997993, 0.23854921580020058], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03702791593969, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.87584601342678, "probs": [0.7887645476620578, 0.2112354523379421], "top": "react", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79003511555493, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16139906086028, "probs": [0.4921584130878123, 0.5078415869121877], "top": "ordinary", "agreement": 0.5, "input_tokens": 191}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56618218868971, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.10097904317081, "probs": [0.293524032474578, 0.7064759675254221], "top": "ordinary", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80322404019535, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77868392318487, "probs": [0.5156793119007109, 0.484320688099289], "top": "react", "agreement": 0.5, "input_tokens": 186}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48462091572583, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64348312653601, "probs": [0.6302458313412554, 0.3697541686587445], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19901983626187, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73358416184783, "probs": [0.7318060925310856, 0.2681939074689144], "top": "react", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.01806509308517, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0666078850627, "probs": [0.5469149637193962, 0.45308503628060387], "top": "react", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.54256308078766, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9168761279434, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.69583300128579, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90122501924634, "probs": [0.44532390802961846, 0.5546760919703815], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.01825890317559, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89125590957701, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.74331392161548, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06133799627423, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "agreement": 1.0, "input_tokens": 195}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67149288579822, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77398400381207, "probs": [0.2501313189948441, 0.749868681005156], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.55322097241879, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04088801518083, "probs": [0.17696061199867663, 0.8230393880013234], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27876789309084, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84461504220963, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.8658900372684, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97037891112268, "probs": [0.2870615213045163, 0.7129384786954838], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.64604691974819, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.2792119961232, "probs": [0.4145813330827104, 0.5854186669172896], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.53519508615136, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.50798603147268, "probs": [0.44532391331269805, 0.554676086687302], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.92307091504335, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.713748017326, "probs": [0.35525721497515955, 0.6447427850248404], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.62807588651776, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.09194500744343, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.95268115215003, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.23944097198546, "probs": [0.8269315768853935, 0.12075476980815658, 0.05231365330644987], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.83213090710342, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.75483091548085, "probs": [0.9608203282304785, 0.03917967176952151], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.76461918465793, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.58605408854783, "probs": [0.9620325019409397, 0.02528607877990563, 0.012681419279154706], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.28979702293873, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.28705708682537, "probs": [0.06890580795431694, 0.931094192045683], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.4181550797075, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.99289800040424, "probs": [0.8320645593484999, 0.04643111322039759, 0.12150432743110248], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.01373415254056, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.01612417399883, "probs": [0.9544700034284429, 0.04552999657155723], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.08799300901592, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.85405596531928, "probs": [0.9527150999124248, 0.019483546419057494, 0.027801353668517746], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.2643119841814, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.85603787750006, "probs": [0.12172004535178665, 0.8782799546482134], "top": "holds", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.32225104235113, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.85101999156177, "probs": [0.0645643144235774, 0.035205281163747985, 0.9002304044126745], "top": "tavern", "agreement": 1.0, "input_tokens": 400}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.19203609228134, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.08493517525494, "probs": [0.1862839992992625, 0.8137160007007376], "top": "elsewhere", "agreement": 1.0, "input_tokens": 360}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7175230383873, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.42717601917684, "probs": [0.36775256751831553, 0.024771247006193476, 0.6074761854754909], "top": "tavern", "agreement": 0.6666666666666666, "input_tokens": 412}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.485391035676, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.44636507332325, "probs": [0.0796746792375593, 0.9203253207624406], "top": "holds", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56509184278548, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.88804099522531, "probs": [0.08290919036094808, 0.8994500391535621, 0.017640770485489753], "top": "meet_wren", "agreement": 1.0, "input_tokens": 403}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08378704637289, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.001200100407, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 363}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.36108912527561, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.47540711984038, "probs": [0.6296525959845715, 0.35799812782967866, 0.012349276185749847], "top": "stay", "agreement": 0.6666666666666666, "input_tokens": 415}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.49111799709499, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.94767908751965, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 351}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63391192443669, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.78367705643177, "probs": [0.5514055185500297, 0.41145265329172886, 0.03714182815824141], "top": "stay", "agreement": 1.0, "input_tokens": 404}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.52265006117523, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.83850712701678, "probs": [0.9673224283553485, 0.03267757164465143], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52882101945579, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.98393003642559, "probs": [0.9603331749383929, 0.028023658411333054, 0.011643166650274083], "top": "stay", "agreement": 1.0, "input_tokens": 416}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1606181319803, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.748415870592, "probs": [0.7439389594906934, 0.2560610405093065], "top": "moved", "agreement": 1.0, "input_tokens": 345}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47655100561678, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.73777798749506, "probs": [0.329927824572513, 0.5565122407492623, 0.11355993467822469], "top": "meet_wren", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.16289800964296, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87464896216989, "probs": [0.02983020500731863, 0.9701697949926814], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.29106787033379, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.08253102004528, "probs": [0.7871268373024943, 0.18209101999191218, 0.030782142705593513], "top": "stay", "agreement": 1.0, "input_tokens": 410}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2692292034626, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.7843469940126, "probs": [0.04979902878533396, 0.950200971214666], "top": "holds", "agreement": 1.0, "input_tokens": 334}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38704881072044, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.42668411694467, "probs": [0.9109478037576992, 0.057628718316645906, 0.03142347792565483], "top": "stay", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42142880149186, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.79346899688244, "probs": [0.9719328658769364, 0.028067134123063546], "top": "same", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.41697509400547, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.36130196973681, "probs": [0.9710720272034978, 0.01715460731299303, 0.01177336548350914], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.11364504136145, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.6876010671258, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.31203296780586, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.73779610544443, "probs": [0.1337836073050901, 0.8426345520770127, 0.0235818406178973], "top": "meet_wren", "agreement": 1.0, "input_tokens": 407}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.26032399199903, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.3173558730632, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.5187151003629, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.52179399505258, "probs": [0.7852826544019835, 0.20168841824349107, 0.013028927354525465], "top": "stay", "agreement": 1.0, "input_tokens": 419}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.05481385998428, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.50320711359382, "probs": [0.05776707671018535, 0.9422329232898147], "top": "holds", "agreement": 1.0, "input_tokens": 336}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.63797700963914, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.76186100393534, "probs": [0.706475960089318, 0.29352403991068193], "top": "stay", "agreement": 1.0, "input_tokens": 370}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90194609574974, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.56751297973096, "probs": [0.9631149772558724, 0.036885022744127596], "top": "same", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.96058705449104, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.21833190508187, "probs": [0.9583890947004433, 0.04161090529955677], "top": "stay", "agreement": 1.0, "input_tokens": 381}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.86197403259575, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.73779599927366, "probs": [0.05608316364845092, 0.9439168363515491], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.55039091594517, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87953817285597, "probs": [0.5156793134278208, 0.4843206865721793], "top": "stay", "agreement": 0.5, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.23427814804018, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.74265703745186, "probs": [0.9471470205154783, 0.05285297948452184], "top": "same", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.10826601274312, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.95091915875673, "probs": [0.9471470206637833, 0.05285297933621657], "top": "stay", "agreement": 1.0, "input_tokens": 389}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.15669688209891, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28044094517827, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 340}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.59065292216837, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.46572291478515, "probs": [0.7614507825686178, 0.23854921743138224], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43584988079965, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.88489816710353, "probs": [0.9583890946533792, 0.04161090534662085], "top": "same", "agreement": 1.0, "input_tokens": 353}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97779696434736, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.18967208266258, "probs": [0.9516645320340041, 0.04833546796599583], "top": "stay", "agreement": 1.0, "input_tokens": 385}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48448098450899, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.21024995669723, "probs": [0.4921584129984156, 0.5078415870015844], "top": "holds", "agreement": 1.0, "input_tokens": 333}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25595794059336, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.65167497098446, "probs": [0.07094578575062087, 0.9290542142493791], "top": "departure", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.12859608605504, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.42042291909456, "probs": [0.0820053924809899, 0.9179946075190101], "top": "elsewhere", "agreement": 1.0, "input_tokens": 346}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72177382744849, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.8624181356281, "probs": [0.1724380702954841, 0.8275619297045159], "top": "departure", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66487304493785, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.83396692946553, "probs": [0.26819391433993306, 0.7318060856600669], "top": "holds", "agreement": 1.0, "input_tokens": 339}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80802500806749, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.86173803731799, "probs": [0.04833546845435023, 0.9516645315456497], "top": "departure", "agreement": 1.0, "input_tokens": 373}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22348807752132, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.57432397454977, "probs": [0.06310762735784174, 0.9368923726421583], "top": "elsewhere", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.14892592653632, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.53668317571282, "probs": [0.17243806760921704, 0.8275619323907829], "top": "departure", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.24607707373798, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.83271802589297, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 337}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84839504770935, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87052809260786, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "agreement": 1.0, "input_tokens": 371}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.49153994396329, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.02488898858428, "probs": [0.9608203281857868, 0.039179671814213245], "top": "same", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11817598342896, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98541907221079, "probs": [0.9439168372028508, 0.05608316279714923], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89450599253178, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36770214512944, "probs": [0.07740461992444837, 0.9225953800755516], "top": "holds", "agreement": 1.0, "input_tokens": 343}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0111271366477, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63476517982781, "probs": [0.6589789156058212, 0.3410210843941787], "top": "stay", "agreement": 1.0, "input_tokens": 377}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.8593928720802, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43625191040337, "probs": [0.9596222367905244, 0.040377763209475684], "top": "same", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.68732397258282, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98649894073606, "probs": [0.9422329206296081, 0.05776707937039187], "top": "stay", "agreement": 1.0, "input_tokens": 388}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.1833368986845, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.73719599470496, "probs": [0.061278160236647276, 0.9387218397633527], "top": "holds", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73247402533889, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.7128149010241, "probs": [0.8910699865839312, 0.10893001341606875], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.090656099841, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76285695843399, "probs": [0.9663161960398492, 0.03368380396015082], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8460560310632, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.9207880795002, "probs": [0.9673224283958624, 0.03267757160413767], "top": "stay", "agreement": 1.0, "input_tokens": 394}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43080001696944, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.81554699875414, "probs": [0.8527907664515249, 0.14720923354847515], "top": "moved", "agreement": 1.0, "input_tokens": 348}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76110404543579, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 77.26746192201972, "probs": [0.04161090472473705, 0.9583890952752631], "top": "outside", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04029709100723, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76070700027049, "probs": [0.05608316279714923, 0.9439168372028508], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.96910383552313, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.99359887838364, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "agreement": 1.0, "input_tokens": 390}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.419128915295, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97074887901545, "probs": [0.07094578723996812, 0.9290542127600319], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.15635694935918, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63877498917282, "probs": [0.8040183777913431, 0.19598162220865686], "top": "stay", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.100695990026, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31223117560148, "probs": [0.959622237525547, 0.040377762474453], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25087804161012, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.79137684963644, "probs": [0.9583890941641539, 0.041610905835846104], "top": "stay", "agreement": 1.0, "input_tokens": 396}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11810683272779, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.33206112869084, "probs": [0.7781221683337017, 0.22187783166629826], "top": "moved", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30091800354421, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63838406652212, "probs": [0.21123545145303588, 0.7887645485469642], "top": "outside", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0087651386857, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43410195223987, "probs": [0.03688502246073314, 0.9631149775392668], "top": "elsewhere", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.09778607264161, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.8235871065408, "probs": [0.755706186939545, 0.244293813060455], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.92706300318241, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.23281892947853, "probs": [0.12172004614068278, 0.8782799538593171], "top": "holds", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47922017611563, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.35892093740404, "probs": [0.8879876230204682, 0.11201237697953184], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.08879601582885, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.69162498414516, "probs": [0.9558139330893122, 0.04418606691068789], "top": "same", "agreement": 1.0, "input_tokens": 366}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.8522628992796, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71660608239472, "probs": [0.9631149782667298, 0.0368850217332701], "top": "stay", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.094956016168, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36917107366025, "probs": [0.09733498488397495, 0.9026650151160249], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.10607600770891, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.62737490236759, "probs": [0.8319924818822048, 0.16800751811779524], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.13582594320178, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.53120304085314, "probs": [0.9471470206637833, 0.05285297933621657], "top": "same", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.52480895072222, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76193611323833, "probs": [0.955813931654481, 0.044186068345518914], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.05968496762216, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}], "seconds": 50.1, "failures": 0}, "repeat": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23254916258156, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 34.15}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.028327118605375, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.03}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03186591155827, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.03}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.162038031965494, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.23}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.879763938486576, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 33.94}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05266700126231, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.12}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86464414931834, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 33.88}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47336193919182, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.55}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16909815557301, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.09}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13988699018955, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.19}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09222702309489, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.16}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16548811644316, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 34.2}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02711500227451, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.08}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1583579108119, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.2}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80704301223159, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 33.89}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.313819862902164, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.37}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94368596188724, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.04}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.175297969952226, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.19}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95994499139488, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.05}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.291509099304676, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.37}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94928507320583, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 33.95}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89924511127174, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 33.99}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94680496118963, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.01}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51171287149191, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.56}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78108309395611, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 33.8}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.217289209365845, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.29}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37996006384492, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.4}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.123268004506826, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.22}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67255211621523, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 33.79}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.406891118735075, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 34.4}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.506030010059476, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 33.6}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16443805024028, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.23}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59661998972297, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 33.66}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05005603842437, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.0}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01518615707755, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.09}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56603295914829, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.55}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08721697330475, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.1}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09544683806598, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.21}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.922294879332185, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 33.91}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05770708620548, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.13}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.716622998937964, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 33.72}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0760359801352, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.18}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.112477933987975, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.17}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50309208407998, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.56}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76608297973871, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 33.83}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.208828842267394, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.24}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56734013929963, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 33.64}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06947714090347, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.1}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65739407390356, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.73}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.596262991428375, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.6}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.999445874243975, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.07}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38411096855998, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.46}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.116636987775564, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.16}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1012669056654, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.2}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93158505484462, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.02}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20871894620359, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.24}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.114576902240515, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.13}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.106527015566826, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.22}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8676339276135, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.05}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2333291452378, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.3}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.712421802803874, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 33.89}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49992209486663, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.7}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65614197961986, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 33.74}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0118860155344, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.23}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57118999399245, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 33.9}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19130903482437, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.45}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.290109088644385, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.4}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366561125963926, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.43}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31977893039584, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.45}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.452901946380734, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.57}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80878388695419, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.02}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17176802456379, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.27}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97552508674562, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.14}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03739587217569, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.19}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.830243887379766, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 33.89}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20806911587715, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.55}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.533589009195566, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 34.03}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25649998523295, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.75}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75839211791754, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.22}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05988684669137, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.58}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.710422951728106, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.21}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05339599400759, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.55}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53504000976682, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.05}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12036716751754, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.54}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.691071931272745, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.19}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20709797926247, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.69}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83449397794902, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.35}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.119027925655246, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.56}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00619516894221, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.49}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26585884578526, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.6}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0895869564265, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.41}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.283570040017366, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 34.68}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.995964823290706, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.43}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.386670941486955, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.66}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.835734033957124, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 33.92}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15450712479651, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.32}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87222511507571, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.01}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21134806983173, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.53}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.723172990605235, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.21}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71010506898165, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 35.2}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89850401133299, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.23}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31600008904934, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.63}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.025655852630734, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.44}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59711282327771, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.9}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.006615893915296, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.39}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18516789376736, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.5}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01397590339184, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.29}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.299609975889325, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.65}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05335594713688, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.42}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1631879825145, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.51}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.970836110413074, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.28}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24760003387928, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.64}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01944509521127, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.31}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.265409948304296, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.59}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84754390269518, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.16}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43252111598849, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.8}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91513487137854, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.22}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.264188984408975, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.64}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80208302102983, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.25}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41681110113859, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.76}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.902745021507144, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.32}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3522499576211, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.67}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07059612497687, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.37}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.420832086354494, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.76}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.055006017908454, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.39}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.182188127189875, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.51}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08865703083575, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.39}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3034100048244, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.63}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79778310470283, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.12}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15852787718177, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.49}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95087507367134, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.32}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33392991125584, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.74}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.754903150722384, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.14}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.285798927769065, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.68}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74098290503025, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.24}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96481604129076, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.49}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11859695613384, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.23}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44849096983671, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.77}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59725096076727, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.06}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11113706231117, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.61}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86432493850589, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.36}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99478507041931, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.49}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87612502835691, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.39}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84546402655542, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.32}], "seconds": 5.4, "failures": 0}, "negative": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.882923915982246, "probs": [0.06566437538338366, 0.8598927385077062, 0.07444288610891006], "top": "insufficient", "option_logits": null, "prompt_sha256": "2111f6636d10660f740e0f032908924d0640e60a712060af19c6e8972d95e5c9", "input_tokens": 204, "server_ms": 34.37}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.292150082066655, "probs": [0.2945662267563794, 0.03956295740237862, 0.665870815841242], "top": "contradicted", "option_logits": null, "prompt_sha256": "ed5aa8d62ce8778e40b8c51f1d1fe90b1f392ab252247908adc4e84b2546ab08", "input_tokens": 202, "server_ms": 34.76}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.66695184074342, "probs": [0.9241406947470572, 0.051569175242271295, 0.024290130010671554], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6f696700120ee981b3e3e0dc237f67896db84867803273e124be96bfd72fecc", "input_tokens": 196, "server_ms": 34.16}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.28761896491051, "probs": [0.11215130215053613, 0.052825544241278395, 0.8350231536081855], "top": "contradicted", "option_logits": null, "prompt_sha256": "ef4b71a8ae0a1bc3711450f77606ddabd2be62c9c2b6587e1fcacbccfcb9cb00", "input_tokens": 196, "server_ms": 34.73}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.765523022040725, "probs": [0.15307015041158983, 0.014109646797217717, 0.8328202027911924], "top": "insufficient", "option_logits": null, "prompt_sha256": "215840810bc2483aa303cd79f5d4f92093a49aa9957dfa697f20989aa1390d55", "input_tokens": 209, "server_ms": 34.25}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.00892603956163, "probs": [0.03329916574926079, 0.8166100641505288, 0.1500907701002104], "top": "supported", "option_logits": null, "prompt_sha256": "fc0d6607f95e1b53fe2cfa889a2cf1bfed2a95e2da0424c85742050fd8b5cde7", "input_tokens": 206, "server_ms": 34.5}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.63306194357574, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "supported", "option_logits": null, "prompt_sha256": "f3bf4ddaa8f4178e0b5f59c09fffe0c07bfd4bd34d616bbf9577361257ca1afa", "input_tokens": 208, "server_ms": 34.07}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.06369595602155, "probs": [0.3541639151292647, 0.5160413123689802, 0.1297947725017551], "top": "contradicted", "option_logits": null, "prompt_sha256": "e5578ff7fa77b4e9f8593ede9b8e31ecccdca126f02444f3c94cdd0bb01ee6ca", "input_tokens": 200, "server_ms": 34.51}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.70328203588724, "probs": [0.017835936327419456, 0.8721495360638406, 0.11001452760873995], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b2ae5222808313b49d4b84a395fe3c5b48345949fa231e2c8fba100ed470a94", "input_tokens": 194, "server_ms": 34.14}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.26375917904079, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b42ed76cec9fa4a467694c0069d312883b296f2ee2385059235881dee283be", "input_tokens": 194, "server_ms": 34.72}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.780972965061665, "probs": [0.03186734083364, 0.0247947114952375, 0.9433379476711226], "top": "contradicted", "option_logits": null, "prompt_sha256": "d2c60f4c3e6615ad87fdb4cd474622ea0a7f3f3b3fd7df582b5539d02a9dbd1b", "input_tokens": 193, "server_ms": 34.3}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.11374709382653, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "contradicted", "option_logits": null, "prompt_sha256": "89f547d1f4dbfccb3c2e3443f18b4576344bd16efeaec8108d2febb0c913ec9e", "input_tokens": 192, "server_ms": 34.55}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.77100292406976, "probs": [0.03540454160362492, 0.924457767599498, 0.04013769079687701], "top": "contradicted", "option_logits": null, "prompt_sha256": "6beeb2f79fa6330dc68f2b769cf2779997d4b57c459bf40113bbc11a80353072", "input_tokens": 205, "server_ms": 34.28}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.98971588164568, "probs": [0.028539740573724572, 0.7452090392508762, 0.22625122017539928], "top": "insufficient", "option_logits": null, "prompt_sha256": "b094a32aab8f6f92e09b2405572effe6bec478d6d3b5985cdeb71dab794ed591", "input_tokens": 205, "server_ms": 34.41}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.95827513001859, "probs": [0.026796810283021767, 0.844600378585805, 0.12860281113117333], "top": "insufficient", "option_logits": null, "prompt_sha256": "4e199755434441474e22e920ad5f8587827e30212d91945271ceadbb26337008", "input_tokens": 199, "server_ms": 34.43}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.26776922121644, "probs": [0.1303439116251751, 0.013621023271142042, 0.8560350651036829], "top": "contradicted", "option_logits": null, "prompt_sha256": "d75aa0e009db26e5e38e3bfd553573eccf95a0526360da23bada050f23e908f8", "input_tokens": 194, "server_ms": 34.7}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.7253830190748, "probs": [0.1369403800499024, 0.018392355065480374, 0.8446672648846173], "top": "supported", "option_logits": null, "prompt_sha256": "9e994209df771b9f8829dd54c36907e41ed241a507cce230c7e3505302a9fa1e", "input_tokens": 198, "server_ms": 34.26}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.44961088523269, "probs": [0.7651520472632017, 0.10276696393112311, 0.13208098880567515], "top": "contradicted", "option_logits": null, "prompt_sha256": "45785396c292fe5cbe62d99a45714d49c993d6f5b5783769cec1795f7b84f35f", "input_tokens": 198, "server_ms": 34.8}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.95624601095915, "probs": [0.9249067812606918, 0.05495367443647756, 0.020139544302830666], "top": "contradicted", "option_logits": null, "prompt_sha256": "490eb806b0921b46aa816c18534ea40c0719e89eb76c34feb4e89d55c921bf36", "input_tokens": 194, "server_ms": 34.4}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.182107800617814, "probs": [0.0190993007707109, 0.7736996936098133, 0.2072010056194758], "top": "contradicted", "option_logits": null, "prompt_sha256": "7e29e327fc2aef321d690607a7c7347871c7eb3dd07ac6d45327aab094603d91", "input_tokens": 194, "server_ms": 34.64}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.70512209646404, "probs": [0.021206655711830842, 0.5200601388697108, 0.45873320541845836], "top": "insufficient", "option_logits": null, "prompt_sha256": "ffb27e8dfc1d8298a601b7f483c42104d68f6e8ebfea086896afd140756b8c6c", "input_tokens": 211, "server_ms": 34.22}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.09609713405371, "probs": [0.9097361205627902, 0.05076536678270011, 0.039498512654509606], "top": "supported", "option_logits": null, "prompt_sha256": "e7855c00f037441fcdc5ff3fefaa61ddaeb0e79e09ba6c0c6043c767ee6d98ca", "input_tokens": 211, "server_ms": 34.53}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.713691893965006, "probs": [0.9223434223370909, 0.04004589025644397, 0.03761068740646525], "top": "supported", "option_logits": null, "prompt_sha256": "1a0c7ecb563009263a7969e56c6e00fc34f99b8ad08b4db579ed9e2bc78f4c6f", "input_tokens": 201, "server_ms": 34.23}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.22501895390451, "probs": [0.1051237595843464, 0.007539451491827663, 0.8873367889238258], "top": "contradicted", "option_logits": null, "prompt_sha256": "610dae3d86ea66ae02b343c18c1478ef66be2619fe3a6c03ff05a434fac307d6", "input_tokens": 197, "server_ms": 34.74}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.568820010870695, "probs": [0.7409240241997488, 0.18635729477403776, 0.07271868102621346], "top": "insufficient", "option_logits": null, "prompt_sha256": "52c15cca48120ff357156a28827735982f0407b36b9806dc79a94193e52ed2a4", "input_tokens": 212, "server_ms": 34.1}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.00299700722098, "probs": [0.9070425995126343, 0.07852475778895956, 0.01443264269840612], "top": "prohibited", "option_logits": null, "prompt_sha256": "5df6c8de75514db23d6e53210ba8920e11a83a08e31ffd76867eb6d789db3f29", "input_tokens": 218, "server_ms": 34.52}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.838924046605825, "probs": [0.04968326826324694, 0.890344493113758, 0.05997223862299509], "top": "insufficient", "option_logits": null, "prompt_sha256": "3dc91e8f64358377c40956ef40870819732d6e66fc8ed44530ee8b7b1e32f1e6", "input_tokens": 203, "server_ms": 34.29}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.92587395198643, "probs": [0.09374299764974467, 0.009200493913293934, 0.8970565084369614], "top": "prohibited", "option_logits": null, "prompt_sha256": "866701419f4e54bef33f2151b0e4c0c830728dbfd94c6ba4a3fbd92b438151a2", "input_tokens": 209, "server_ms": 34.38}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.738652968779206, "probs": [0.20575303419639995, 0.025954061735184695, 0.7682929040684153], "top": "prohibited", "option_logits": null, "prompt_sha256": "6c14eaaad2ab386d458a6458fce2b79bcd81ecbfc3c66679c950d808dd0c2a3f", "input_tokens": 209, "server_ms": 34.28}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.79768298752606, "probs": [0.8315971903507323, 0.10489918128594154, 0.06350362836332618], "top": "permitted", "option_logits": null, "prompt_sha256": "d8b705bc82e40643b8dcd135b43f67a8e3671ecc2f28e81b48de6aa1cbb84792", "input_tokens": 214, "server_ms": 34.3}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.95413609966636, "probs": [0.258643657502053, 0.6628301324407543, 0.07852621005719278], "top": "insufficient", "option_logits": null, "prompt_sha256": "5000e3b05641e28823047872c8bd19f4604d76e0fa38c23bdad3d8e7be7bce61", "input_tokens": 209, "server_ms": 34.49}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.17563790269196, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "prohibited", "option_logits": null, "prompt_sha256": "97ddaad2b5074dab6f31a5a7a8ded12c3e51123aa5d819f19c006ba123fde089", "input_tokens": 209, "server_ms": 34.67}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.768823163583875, "probs": [0.09658208517581993, 0.868022296339126, 0.03539561848505407], "top": "insufficient", "option_logits": null, "prompt_sha256": "5783bf781c97c761780f8d8454e12c8895251116d4d5ebe264401d9d9dcae0a1", "input_tokens": 208, "server_ms": 34.29}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.11969708092511, "probs": [0.0651099549445377, 0.02705164492548852, 0.9078384001299739], "top": "prohibited", "option_logits": null, "prompt_sha256": "a92081d320caa09114cab155b336249985672a93e86328503ed4397aa61c3293", "input_tokens": 211, "server_ms": 34.46}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.08112589083612, "probs": [0.049132112383896635, 0.729412594713007, 0.22145529290309635], "top": "insufficient", "option_logits": null, "prompt_sha256": "2b0f13909e189564a5e5099e9094600d4804ae6ef7ef59e7435f85302977a0c0", "input_tokens": 202, "server_ms": 34.56}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.12230804376304, "probs": [0.1638594512396548, 0.04974899459992729, 0.7863915541604178], "top": "prohibited", "option_logits": null, "prompt_sha256": "a4ffa08e79c520845fb80c116412fe4c48304f5f003b514bbcbea0a7c61cfe7b", "input_tokens": 204, "server_ms": 34.61}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.84355295263231, "probs": [0.06162034871506134, 0.8591822512326693, 0.07919740005226927], "top": "insufficient", "option_logits": null, "prompt_sha256": "f51b6f4e075e67f76ff574d496ba2f9c19cc2ca38a2f56d989791df67251d344", "input_tokens": 214, "server_ms": 34.33}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.06784709356725, "probs": [0.7621344563262837, 0.23138990123980285, 0.006475642433913465], "top": "insufficient", "option_logits": null, "prompt_sha256": "baf1674ea5ff54f508700c815711c06e9a50808663840042c147d883c72021ed", "input_tokens": 214, "server_ms": 34.56}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.64665203541517, "probs": [0.14398191537109406, 0.021923360098142518, 0.8340947245307634], "top": "insufficient", "option_logits": null, "prompt_sha256": "8854be3865426f818cb7189838d63dc61b9b64da67d13087c8e6ccc2a708a8c0", "input_tokens": 213, "server_ms": 34.1}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.215728079900146, "probs": [0.2129289153471605, 0.545676239155193, 0.24139484549764645], "top": "permitted", "option_logits": null, "prompt_sha256": "913468ed99926586c9292a6dabcee9b083a7b1b1c719b6bad31fda7f7f045a2e", "input_tokens": 212, "server_ms": 34.73}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.676913034170866, "probs": [0.15943768499721103, 0.025848589203191098, 0.814713725799598], "top": "insufficient", "option_logits": null, "prompt_sha256": "acfb724b1ffcab3d0820c0a1f4cdb189396280c1faf4e622ce1f306cad1bf45e", "input_tokens": 216, "server_ms": 34.16}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.085916148498654, "probs": [0.9471510047644319, 0.04112297491157286, 0.011726020323995183], "top": "insufficient", "option_logits": null, "prompt_sha256": "7a9df7814cddba377022c022111d998db3d8a2ff48c109eb7f1aac0f5c151319", "input_tokens": 216, "server_ms": 34.57}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.89005505293608, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "ecd27e22648cba485966b925a1b12fd5a888331b66634b0f8919798224cf8072", "input_tokens": 208, "server_ms": 34.37}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.348940037190914, "probs": [0.2729138208779564, 0.3294318709588002, 0.3976543081632435], "top": "insufficient", "option_logits": null, "prompt_sha256": "86759c52a7cc961194391587ca2df8d3998c59ec236637e4acc083f151ad5242", "input_tokens": 215, "server_ms": 34.78}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.823054077103734, "probs": [0.5472741025611921, 0.331307576591957, 0.12141832084685092], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ee81091d38ca686274af8a9057ae962583e07c84d4a7c8913246f20a9adc153", "input_tokens": 209, "server_ms": 34.16}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.082756869494915, "probs": [0.10569064817366915, 0.056437743537509265, 0.8378716082888217], "top": "insufficient", "option_logits": null, "prompt_sha256": "d890e328d92ed0294fd0ca0f45891d715f1b53fae8d48848ee13c936721cc9f2", "input_tokens": 212, "server_ms": 34.38}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.88425407744944, "probs": [0.745864682612535, 0.0779433517872937, 0.17619196560017134], "top": "permitted", "option_logits": null, "prompt_sha256": "866975030f9845962316982b48126dfbaa42cd89b8d3719305c5c1cd2a7ce35c", "input_tokens": 209, "server_ms": 34.21}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.19308786094189, "probs": [0.30671712087126574, 0.5394587819774863, 0.153824097151248], "top": "permitted", "option_logits": null, "prompt_sha256": "33ed49b156b02a8251a71c89906619f22d3cff122755a441bc701809f452ec03", "input_tokens": 210, "server_ms": 34.56}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.12631715461612, "probs": [0.9411304198850982, 0.03837677130689544, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "26a5967a0d4f86a5e648fdf739a14ae8463a48c7dc0319e4b9026afcfc70ce49", "input_tokens": 200, "server_ms": 34.48}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.21683891490102, "probs": [0.2191677303480843, 0.5616645393038313, 0.2191677303480843], "top": "insufficient", "option_logits": null, "prompt_sha256": "5cb564ef00f9a81eb96ac41fe01780b4b9ffdcb60d7bc8217b9d89860c279ba3", "input_tokens": 201, "server_ms": 34.55}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.06779703497887, "probs": [0.8246471790493917, 0.16138164146161363, 0.013971179488994609], "top": "insufficient", "option_logits": null, "prompt_sha256": "e747a01e40c9029fd1004867b0db224405c02f421c64f74cd748920300cb2000", "input_tokens": 192, "server_ms": 34.59}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.03592601232231, "probs": [0.9136677967042812, 0.07428832300095413, 0.012043880294764588], "top": "B", "option_logits": null, "prompt_sha256": "3d5649d07c298093a047c4d77967454e627c1fd4792f41873c4640c036a163cd", "input_tokens": 192, "server_ms": 34.56}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.79317398928106, "probs": [0.807231387784515, 0.17909249095841714, 0.013676121257067941], "top": "insufficient", "option_logits": null, "prompt_sha256": "88ac81fbf23c9684447ca373863f9cf4c54ca0bd35eb3e960680db4187456d96", "input_tokens": 208, "server_ms": 34.32}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.13940689340234, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "insufficient", "option_logits": null, "prompt_sha256": "7263692449eeef173103012af0a80bd27db0a89dc35bf50d4fed73b80a29f045", "input_tokens": 211, "server_ms": 34.63}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.88157512806356, "probs": [0.046119873005046885, 0.9369780144263187, 0.016902112568634386], "top": "B", "option_logits": null, "prompt_sha256": "4c2c0840c1ad7e181d30f10eb62a4618c3195e936034bc7274fe3c8fc945d786", "input_tokens": 200, "server_ms": 34.37}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.13695681653917, "probs": [0.6742384798146065, 0.29826788550402233, 0.02749363468137107], "top": "B", "option_logits": null, "prompt_sha256": "2d0f22653ae25b1b9ad253802809222e0554c7bd42292a5a1637a04128e5c52f", "input_tokens": 190, "server_ms": 34.67}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.99854598380625, "probs": [0.8763745016908776, 0.09751141040687612, 0.026114087902246324], "top": "insufficient", "option_logits": null, "prompt_sha256": "676f3a162c1d11c22fd5f07e5b57e6b6e6d3a4aae515a7c2965aeac124b616dd", "input_tokens": 203, "server_ms": 34.45}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.888554925099015, "probs": [0.13968031972950484, 0.23073250176366764, 0.6295871785068274], "top": "insufficient", "option_logits": null, "prompt_sha256": "109526bdb944267ab212d1b8ba2f77a3a6c7c147016cf89ae5bc2e0e8e661adc", "input_tokens": 206, "server_ms": 34.4}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.04133606515825, "probs": [0.9673249441344611, 0.025425185481267775, 0.007249870384270957], "top": "insufficient", "option_logits": null, "prompt_sha256": "c4889b691603599fd8dca223a645f5e88b7248a364a9bef64c37a8a9e3ced5eb", "input_tokens": 197, "server_ms": 34.55}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.224018946290016, "probs": [0.49248819366063284, 0.16951261805505752, 0.3379991882843097], "top": "insufficient", "option_logits": null, "prompt_sha256": "5deb35c130c31229df54964339f9faccffa08f3ed2eb38730d86ac9ccd510aa3", "input_tokens": 192, "server_ms": 34.7}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.71655198559165, "probs": [0.1558415388693792, 0.330859461294187, 0.5132989998364338], "top": "insufficient", "option_logits": null, "prompt_sha256": "7c27c7c3361bd04c584a975218ab6b610dcab0f9bf6078cf3870b470410c383c", "input_tokens": 203, "server_ms": 34.18}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.122106878086925, "probs": [0.30165258046080173, 0.13344430167683133, 0.5649031178623669], "top": "B", "option_logits": null, "prompt_sha256": "27dba9d4c38e8b61907893c325f5851e4726ba89fa7ac9e5563a14abe89239b9", "input_tokens": 203, "server_ms": 34.63}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.791404010728, "probs": [0.9480355945752582, 0.038658346366163006, 0.013306059058578745], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f3dd4d0a864d976d5c35a7b088b3aed4b55500043acc8701dd4d35729125db8", "input_tokens": 193, "server_ms": 34.31}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.19082800671458, "probs": [0.22389674812152066, 0.7374540645868362, 0.03864918729164314], "top": "A", "option_logits": null, "prompt_sha256": "164c8915823587dee72ef8800598282728e580a02992fe7a8d8fa557ead4e3b9", "input_tokens": 192, "server_ms": 34.65}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.003136007115245, "probs": [0.9379420530362722, 0.04616732447007806, 0.01589062249364978], "top": "insufficient", "option_logits": null, "prompt_sha256": "d7c03673e95db552913947e72213d55f0938d187ca0d4fd69182cbe141bfda65", "input_tokens": 208, "server_ms": 34.35}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.15844685211778, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "A", "option_logits": null, "prompt_sha256": "b54ee64662a21acf112153e17512f10436c1938df35ab9ad182be1ff0c47c3e0", "input_tokens": 206, "server_ms": 34.44}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.157858023419976, "probs": [0.13579878113602123, 0.8376256951826758, 0.02657552368130303], "top": "A", "option_logits": null, "prompt_sha256": "8aa1b7f158bf57460e6f9f057913e0d01d913f16998cc0a46327e8aa6c93e838", "input_tokens": 200, "server_ms": 34.55}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.42071194574237, "probs": [0.25821814633948337, 0.25821814633948337, 0.4835637073210332], "top": "A", "option_logits": null, "prompt_sha256": "b17c06e9ca0e91accd4cfcb4667c5c3a42687d3209e1ecd98910eb1343a18dc2", "input_tokens": 193, "server_ms": 34.7}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.91124403849244, "probs": [0.12660014777476133, 0.09251256875552054, 0.7808872834697181], "top": "insufficient", "option_logits": null, "prompt_sha256": "324bb2046ede92322b8175d6af2961ad208eba5f52712692465ef944de2193a4", "input_tokens": 206, "server_ms": 34.25}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.335600005462766, "probs": [0.9299397393693557, 0.06263931303365462, 0.007420947596989686], "top": "insufficient", "option_logits": null, "prompt_sha256": "86e7d75bb9b41e2d9de272239185c5cce5c19dc1aac6573a737c31d47ab0563f", "input_tokens": 209, "server_ms": 34.7}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.07154607400298, "probs": [0.37290349783252347, 0.01111172675798736, 0.6159847754094891], "top": "insufficient", "option_logits": null, "prompt_sha256": "a1a07677dc816ad01403f9cff2867d5bfdaff2c29d8a4c2293ce0662a24f33ae", "input_tokens": 194, "server_ms": 34.39}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.177748046815395, "probs": [0.475785940079552, 0.1976777825029813, 0.3265362774174668], "top": "insufficient", "option_logits": null, "prompt_sha256": "82b8973ae0b97456180d256ef08822a11d20365d8039148ed863e9bb75c9bda4", "input_tokens": 192, "server_ms": 34.54}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.086677039042115, "probs": [0.14585095569321507, 0.009226876843769601, 0.8449221674630154], "top": "insufficient", "option_logits": null, "prompt_sha256": "9a672222518ab31da9c3ab2072849ef03b511e8e9d23762f89d33a83feca6ec1", "input_tokens": 216, "server_ms": 34.39}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.23476896993816, "probs": [0.09629593898139954, 0.7633942252849403, 0.14030983573366013], "top": "contradicted", "option_logits": null, "prompt_sha256": "5d2348206184a53d2da714668c8004522fc660c04abc09a4675170278b6d0ce8", "input_tokens": 214, "server_ms": 34.57}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.053456995636225, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "497e689287006b51bd4f178ebc9023050c338fca8abf790eef2112975de7f243", "input_tokens": 209, "server_ms": 34.4}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.25142800249159, "probs": [0.9369780144263187, 0.046119873005046885, 0.016902112568634386], "top": "supported", "option_logits": null, "prompt_sha256": "984a11157ce42cd37ec6209d96b53bf0d293025d6b69161f1a358f9bcc9d57db", "input_tokens": 207, "server_ms": 34.6}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.92340478301048, "probs": [0.6452505747343799, 0.2680863307491953, 0.08666309451642484], "top": "permitted", "option_logits": null, "prompt_sha256": "ea5a71f7bf051b93ca522c5ff3caef6ffdfcb5011f24af51a404b10524b250cb", "input_tokens": 228, "server_ms": 34.2}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.37190109677613, "probs": [0.43020843462499875, 0.1903144997976119, 0.3794770655773894], "top": "permitted", "option_logits": null, "prompt_sha256": "b5ac43b527f17cc117db3df096bff98320c5a248422c7b513eedeba99e619731", "input_tokens": 229, "server_ms": 34.69}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.76335304044187, "probs": [0.056437743537509265, 0.8378716082888217, 0.10569064817366915], "top": "insufficient", "option_logits": null, "prompt_sha256": "be77461c7c49b38137b436f74bcc533cf9fdcc1fbbcabe3acbe8c8c37f2272bd", "input_tokens": 222, "server_ms": 34.1}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.137707928195596, "probs": [0.1110623891201585, 0.008481107191351438, 0.88045650368849], "top": "permitted", "option_logits": null, "prompt_sha256": "12c28c53461194e05d3490b99aecb157c1965b942f5bfee5ef8163543d2bb1d0", "input_tokens": 221, "server_ms": 34.46}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.139806896448135, "probs": [0.7598853370160624, 0.23070704972212444, 0.009407613261813141], "top": "insufficient", "option_logits": null, "prompt_sha256": "0b7bc0317c30767061abcd7c39f9b71942a3c49b230cf334d1e6db7873b4dc61", "input_tokens": 216, "server_ms": 34.46}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.2179988771677, "probs": [0.9718671022270775, 0.019875215614779472, 0.008257682158143088], "top": "B", "option_logits": null, "prompt_sha256": "282f482d7bf4f65f4cc0f8dd6ca05d597f32f80783d49b010173ef79b130ffc5", "input_tokens": 210, "server_ms": 34.61}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.89859388396144, "probs": [0.08284327606718382, 0.0730741626697933, 0.8440825612630228], "top": "A", "option_logits": null, "prompt_sha256": "6bf8f97ede01d029938070b85d9e3aa545cbbbb84007fb406100f0a0a8ffa474", "input_tokens": 209, "server_ms": 34.25}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.05657692626119, "probs": [0.1753450388599366, 0.7903403209860906, 0.03431464015397285], "top": "A", "option_logits": null, "prompt_sha256": "33dc6c8523ece68e76db4eaffbb8a8470964368747977448381041503e3ad36c", "input_tokens": 211, "server_ms": 34.44}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.16525807976723, "probs": [0.03629282833642113, 0.01605512937357001, 0.9476520422900088], "top": "supported", "option_logits": null, "prompt_sha256": "6e4467088790119e033569f3066af7eedf2f72c934657e3a20082f685de045f2", "input_tokens": 226, "server_ms": 34.57}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.248848006129265, "probs": [0.044616056760944024, 0.6623675120954924, 0.2930164311435636], "top": "insufficient", "option_logits": null, "prompt_sha256": "1a718aab5e212abb4604d4e57ad3d7fb8cba756be66eba7d483eec463cb18c6b", "input_tokens": 227, "server_ms": 34.68}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.053906824439764, "probs": [0.04013769079687701, 0.03540454160362492, 0.924457767599498], "top": "contradicted", "option_logits": null, "prompt_sha256": "bafd2feb34ff693920168afa57cc5bc050eea256c82f2e9e071467805a7857f4", "input_tokens": 202, "server_ms": 34.46}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.17854805290699, "probs": [0.21057073282474073, 0.0031468025509083547, 0.7862824646243509], "top": "supported", "option_logits": null, "prompt_sha256": "c4c6394f10ee2df26d3dac75e02000ae56f28719ec24b10add7af6e77f5a4889", "input_tokens": 202, "server_ms": 34.56}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.04769583418965, "probs": [0.10502328955939726, 0.2865713245281423, 0.6084053859124604], "top": "permitted", "option_logits": null, "prompt_sha256": "37bd9a494e79ba1e91bbf2a68704609c66421b3ec11ab8a5bdd36908b17135b1", "input_tokens": 233, "server_ms": 34.41}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.28953981772065, "probs": [0.2680863307491953, 0.08666309451642484, 0.6452505747343799], "top": "insufficient", "option_logits": null, "prompt_sha256": "a115c9f1aaee5b00fa1fd3be52a599afa7112fdcf9eec8871ae98b3d6ef2183f", "input_tokens": 231, "server_ms": 34.63}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.858303938061, "probs": [0.07184163899867897, 0.8298456430479688, 0.09831271795335216], "top": "prohibited", "option_logits": null, "prompt_sha256": "100e5f7a276a412d51e2ddf940777a1ff109ea7803a8291d638760c392064d80", "input_tokens": 216, "server_ms": 34.21}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.58627315983176, "probs": [0.7469915530638082, 0.14618461002062086, 0.106823836915571], "top": "prohibited", "option_logits": null, "prompt_sha256": "6486b8dc13e88f4d5e489f3b3b76c6bb5aeed63f19005132e10380f0181726aa", "input_tokens": 223, "server_ms": 34.93}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.85957402922213, "probs": [0.8638851317960057, 0.051328051461926705, 0.08478681674206763], "top": "B", "option_logits": null, "prompt_sha256": "941cd3d853797aa1a92c8261e9339af72809062c590b26ef2db8bdbbc5d68191", "input_tokens": 222, "server_ms": 34.2}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.415970815345645, "probs": [0.046253998446132885, 0.9397029314342114, 0.014043070119655722], "top": "A", "option_logits": null, "prompt_sha256": "f9c6ae0da8d46bec4df2b4102f3aebb45748add11af2d63a981d879fea6a717c", "input_tokens": 222, "server_ms": 34.73}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.73108294606209, "probs": [0.04067015270682889, 0.7760155022030047, 0.1833143450901663], "top": "insufficient", "option_logits": null, "prompt_sha256": "4df509501a3fa1e95f377f6b2cd7ff2737ebe52dd6e95472a1a35d0fc43e3fdf", "input_tokens": 215, "server_ms": 34.16}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.48367098160088, "probs": [0.05543534738188995, 0.9330136575005004, 0.01155099511760962], "top": "A", "option_logits": null, "prompt_sha256": "d68329bad204407dddf0266c2d55c897537804b87978786e2c29649197957a18", "input_tokens": 201, "server_ms": 34.77}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.062957184389234, "probs": [0.06878691906304032, 0.03042978117418096, 0.9007832997627787], "top": "supported", "option_logits": null, "prompt_sha256": "3fdb716c314c67c93413daa3cd5a38b13bec265f706d90a97c43e5a108104f13", "input_tokens": 223, "server_ms": 34.46}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.24211920052767, "probs": [0.06893683236028562, 0.08321304657583695, 0.8478501210638775], "top": "contradicted", "option_logits": null, "prompt_sha256": "d68195061969b189d71df192da0191d4d56877cf1d8ce0d4479bc8898c998c94", "input_tokens": 219, "server_ms": 34.61}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.873974138870835, "probs": [0.016115540131028547, 0.8390475816237557, 0.14483687824521582], "top": "insufficient", "option_logits": null, "prompt_sha256": "6edd6bca63afcb6fb6d263e3e248e24e556fde2b01d965155729342ef37e411e", "input_tokens": 220, "server_ms": 34.15}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.16873796656728, "probs": [0.2862501142494634, 0.02478133063598072, 0.6889685551145559], "top": "contradicted", "option_logits": null, "prompt_sha256": "d113705714c5c456d527c8466aeda96099d32a8069c13bf0f40b1ff3ed43ad11", "input_tokens": 218, "server_ms": 34.58}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.9465840049088, "probs": [0.1609098519153101, 0.7722359149508133, 0.06685423313387646], "top": "insufficient", "option_logits": null, "prompt_sha256": "19fd16742b6e54b9e2452e09592de10d5fc44c96cab1630e5f7651b1af77d898", "input_tokens": 224, "server_ms": 34.3}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.089018151164055, "probs": [0.3043910694191852, 0.39121786116162954, 0.3043910694191852], "top": "insufficient", "option_logits": null, "prompt_sha256": "ea512df6fc353426e37889a53cae2118b9e832157a4af067cb4a85489a5c83ff", "input_tokens": 221, "server_ms": 34.41}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.91642405465245, "probs": [0.08097077759111387, 0.04060828666999563, 0.8784209357388906], "top": "prohibited", "option_logits": null, "prompt_sha256": "71ad1455eaea177dccc5d9fc4078b19378e2ce1c1d27289d9120a1c445349487", "input_tokens": 221, "server_ms": 34.34}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.866994109004736, "probs": [0.8979988326296015, 0.06048782007179681, 0.041513347298601795], "top": "permitted", "option_logits": null, "prompt_sha256": "31939a3c71e268cd4751d57ccd6a07ff44f1ca1f81c7c2f7bf165ea50668fe0a", "input_tokens": 220, "server_ms": 34.26}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.13070787489414, "probs": [0.9821934763214742, 0.008345422472445314, 0.009461101206080565], "top": "B", "option_logits": null, "prompt_sha256": "13f4ee2ce11c253ccf784222a18f900b57830384e577f4adf0030d9893ed01a9", "input_tokens": 227, "server_ms": 34.56}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.93371522240341, "probs": [0.9584447268648851, 0.039082802533792566, 0.0024724706013222815], "top": "insufficient", "option_logits": null, "prompt_sha256": "c8a6bf0c918df98324f40d3c989bf2d3593c8114ebaba9510a0de7a63d90c43f", "input_tokens": 230, "server_ms": 34.36}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.56995086930692, "probs": [0.8152011530118691, 0.1808606650321593, 0.003938181955971634], "top": "insufficient", "option_logits": null, "prompt_sha256": "d512a04940165a8dc92bcf5e8a905fa84148aca8078cba93d34897ca73ebe5fc", "input_tokens": 223, "server_ms": 34.09}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.01352491043508, "probs": [0.9509567466006987, 0.028336479497072638, 0.020706773902228682], "top": "A", "option_logits": null, "prompt_sha256": "726c7f274cbd758a0855ced1b8e06e30b0888e289eeb459d087a868fb9610ef5", "input_tokens": 210, "server_ms": 34.57}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.83812520466745, "probs": [0.03787087428859695, 0.033405034339472804, 0.9287240913719302], "top": "supported", "option_logits": null, "prompt_sha256": "0032df826a7e49e7be6e1c8d5f96ece8822a858eeb6f5280c55a79fe2e628bbb", "input_tokens": 227, "server_ms": 34.29}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.14827704243362, "probs": [0.04458832792770586, 0.7990410839770169, 0.15637058809527726], "top": "insufficient", "option_logits": null, "prompt_sha256": "c29cf273a3d78467e10369969a81a42df358b37cb972ad44f78d455f8785b7e8", "input_tokens": 224, "server_ms": 34.69}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.686151986941695, "probs": [0.6079293570442296, 0.046423581791889615, 0.34564706116388083], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b9e9fd277a7252eb95c0dea4201d1d75394e8dd03ee09f576f6fc0355881f3", "input_tokens": 220, "server_ms": 34.21}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.88733512535691, "probs": [0.07421350543321283, 0.7101726109045401, 0.2156138836622472], "top": "contradicted", "option_logits": null, "prompt_sha256": "25fc9b80402930a472ddf5fbb9527f22c905747ffefc59e4865da0bcf422e5bb", "input_tokens": 208, "server_ms": 34.44}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.87865403480828, "probs": [0.3596689227288795, 0.5240624893078483, 0.11626858796327226], "top": "prohibited", "option_logits": null, "prompt_sha256": "1a9834724cc432424395c6e3e4989cae7698e00f85fedac3d9a8cd1a698cd265", "input_tokens": 227, "server_ms": 34.33}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.33333991840482, "probs": [0.17244943152985545, 0.1836151222987057, 0.6439354461714388], "top": "permitted", "option_logits": null, "prompt_sha256": "cc59abf9dfd3346910e8d93dc78fe4cd21d0c20962370b801e609b82712c7ea7", "input_tokens": 228, "server_ms": 34.69}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.87125397846103, "probs": [0.11048645425133781, 0.06688603847853027, 0.8226275072701319], "top": "insufficient", "option_logits": null, "prompt_sha256": "45e75124dce70ee83cbbd2df43d7c5f3045b0da83559f8b4772e8cb6fa82dbb0", "input_tokens": 225, "server_ms": 34.23}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.471982184797525, "probs": [0.07600853330506088, 0.8779775900702179, 0.046013876624721124], "top": "permitted", "option_logits": null, "prompt_sha256": "902f5bf4dd52878d854ff660af610beb5cd00553cb6cf9fa5c1e7a5ea010391b", "input_tokens": 218, "server_ms": 34.77}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.96144511923194, "probs": [0.07014840077352269, 0.16883849753435962, 0.7610131016921177], "top": "B", "option_logits": null, "prompt_sha256": "d4a4f3c22ce6befbff78058a364e4617bda379057c95b5bacf2410a162ea27f7", "input_tokens": 221, "server_ms": 34.37}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.96276596747339, "probs": [0.7195969840802272, 0.2637194956142592, 0.016683520305513583], "top": "insufficient", "option_logits": null, "prompt_sha256": "fc0371a2d78013d514b134511d38b37de6e71a39ad2fa9b95b0119945db7a316", "input_tokens": 227, "server_ms": 34.38}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.878474056720734, "probs": [0.6328238128438389, 0.029254643239392513, 0.33792154391676854], "top": "A", "option_logits": null, "prompt_sha256": "5521ebbd2304bdf6049132465a2b8c7b28c862f462f50f98d90efeb483601e64", "input_tokens": 207, "server_ms": 34.32}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.087426986545324, "probs": [0.22351946247571317, 0.12708523693891682, 0.64939530058537], "top": "A", "option_logits": null, "prompt_sha256": "c11ad01ccd57e2b3b849c07b89887124412bce4a8e5506adaeb954f23c598ac7", "input_tokens": 197, "server_ms": 34.46}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.88112506642938, "probs": [0.038658346366163006, 0.013306059058578745, 0.9480355945752582], "top": "supported", "option_logits": null, "prompt_sha256": "8d5e868c2065409c5dcedc10c1d875c102dfbb48d2433b234f0ad0456f1049bc", "input_tokens": 213, "server_ms": 34.28}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.1362071018666, "probs": [0.01953747345276732, 0.9553520377742281, 0.025110488773004678], "top": "contradicted", "option_logits": null, "prompt_sha256": "6ac7738979b2b31bef2f7e2f3d8984b0b2b559ee71fa4d2b728e28884e3183b5", "input_tokens": 212, "server_ms": 34.55}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.059787195175886, "probs": [0.05100292527189731, 0.035003776957469146, 0.9139932977706335], "top": "contradicted", "option_logits": null, "prompt_sha256": "66c4b4c8b3dc52b623b51acf50a373bf625ad5559da7c3bf90c47f56e7c2829b", "input_tokens": 202, "server_ms": 34.44}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.40518097765744, "probs": [0.09842495451008554, 0.01699017293024806, 0.8845848725596664], "top": "insufficient", "option_logits": null, "prompt_sha256": "7db376c4d8c96d2c69fda861429911890ec2d70b0f375e60f65c846f1693e18a", "input_tokens": 203, "server_ms": 34.81}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.83599294163287, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "03449c2d41ee64f00d22b4b6e619aa69ce6d623d00b4972f503f5ed336bba08e", "input_tokens": 218, "server_ms": 34.15}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.275690119713545, "probs": [0.9331701885712249, 0.04890646327936172, 0.017923348149413328], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ed1d2ff423bdb9e622336508299a3180255754a84d1847b9b5ba5d1d5623692", "input_tokens": 221, "server_ms": 34.73}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.06030594184995, "probs": [0.05755147392994642, 0.0327217252711575, 0.9097268007988961], "top": "insufficient", "option_logits": null, "prompt_sha256": "fb0073f867c77a04f9287adbd88276517a2a85e5cf9dcfb04c81ff5ba768e03f", "input_tokens": 215, "server_ms": 34.44}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.17554803006351, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "insufficient", "option_logits": null, "prompt_sha256": "51c4ac799c31ceb752046516252cb8d59c3a3c1654c685c82476530a4d74a689", "input_tokens": 220, "server_ms": 34.6}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.574610974639654, "probs": [0.040967527214491276, 0.8323022347202245, 0.12673023806528433], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a5a9b08e7ca06d1f5808cdcd3270c79b064cd4fed16b4be278fc7d5f58a1d1f", "input_tokens": 206, "server_ms": 34.02}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.30502910912037, "probs": [0.048847636440916636, 0.07578275550521898, 0.8753696080538644], "top": "insufficient", "option_logits": null, "prompt_sha256": "555fd1e019506f048448ae23a6bac4fe7f3e9f2fe60181be02acfdf8225318e5", "input_tokens": 212, "server_ms": 34.7}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.66696301661432, "probs": [0.07319171247064139, 0.2264135372721482, 0.7003947502572104], "top": "insufficient", "option_logits": null, "prompt_sha256": "1fc713f5dfc86b0d90f75094b087d920fbfeb9830fb4eec74f78d2f1d67fbf66", "input_tokens": 203, "server_ms": 34.1}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.286919075995684, "probs": [0.9387496456243781, 0.055776151759141294, 0.005474202616480596], "top": "B", "option_logits": null, "prompt_sha256": "77e6633b56fd111f3cf38ff30d1c308c084b3de608efb9682cf02409dc31c9db", "input_tokens": 191, "server_ms": 34.69}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.889794981107116, "probs": [0.46903312718147117, 0.38856479455009935, 0.14240207826842943], "top": "supported", "option_logits": null, "prompt_sha256": "83104242d8900e7e51e6ca6f99d29588121dd7153c0fda9d487e3eafef91196e", "input_tokens": 224, "server_ms": 34.29}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.33570105396211, "probs": [0.7720397533404616, 0.11755438599864407, 0.11040586066089443], "top": "insufficient", "option_logits": null, "prompt_sha256": "66f0b52c1eee120f16069ed75fbd7bfc556b5a3dc09c27f809799b32ae74e959", "input_tokens": 221, "server_ms": 34.71}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.113475846126676, "probs": [0.014236196905258059, 0.6140385266223682, 0.3717252764723737], "top": "insufficient", "option_logits": null, "prompt_sha256": "fac76f29223ea6573fb84bb8cb70a27292765e7f4f90ebb19001e8a0b7bf0533", "input_tokens": 229, "server_ms": 34.52}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.13076701387763, "probs": [0.06291220787817095, 0.0030966605414949147, 0.9339911315803341], "top": "supported", "option_logits": null, "prompt_sha256": "89df7654ae643d961d5bd0a1cf472717b674fe14a8f5377a6918c1103303e14f", "input_tokens": 226, "server_ms": 34.53}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.14467701502144, "probs": [0.5216500126916775, 0.26161676918070537, 0.21673321812761712], "top": "prohibited", "option_logits": null, "prompt_sha256": "73a309dd6a1605e1d928e2a4947516f3ae54fda513d77cea9d4fdf892dbbebb5", "input_tokens": 226, "server_ms": 34.51}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.416951963678, "probs": [0.843310768581531, 0.1455727910071266, 0.01111644041134243], "top": "prohibited", "option_logits": null, "prompt_sha256": "97510bceb98f70589de7351d3f810c2e4c175b03bbf6ddec55eca00ef9c199ee", "input_tokens": 215, "server_ms": 34.81}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.89853404648602, "probs": [0.42929779219364844, 0.486689574582582, 0.08401263322376958], "top": "permitted", "option_logits": null, "prompt_sha256": "5a1d1ca84275c476cd245cafcd0e1fd36598a6dae3c643b8ea9d60c5b57d6535", "input_tokens": 225, "server_ms": 34.28}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.38109115511179, "probs": [0.19560247361171754, 0.23611000151056785, 0.5682875248777147], "top": "permitted", "option_logits": null, "prompt_sha256": "fec6fd7a8c54ee7856b56b8cc85aa3ff4cb617820fd52f7faf204e2efb4a86ff", "input_tokens": 224, "server_ms": 34.83}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.941925063729286, "probs": [0.9323898466010938, 0.02450695094155103, 0.04310320245735524], "top": "B", "option_logits": null, "prompt_sha256": "fcc6cca56205c973fe5ee42de7bbca225406234b5346f63e55af7d8c5f115d9a", "input_tokens": 213, "server_ms": 34.23}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.31670998968184, "probs": [0.643833389199536, 0.3438005390976864, 0.012366071702777563], "top": "insufficient", "option_logits": null, "prompt_sha256": "176f3f577d5802518ba6bb8a36039f8c2c7fa4991aac2e236870e0189d0b31a6", "input_tokens": 218, "server_ms": 34.62}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.94967506453395, "probs": [0.020042268594912593, 0.21743089555343206, 0.7625268358516554], "top": "B", "option_logits": null, "prompt_sha256": "8494413c37184f76cf07859eef5fe11f7373b1acb0da7cbd005226a5bb00610d", "input_tokens": 209, "server_ms": 34.34}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.151667987927794, "probs": [0.33058297459294167, 0.050336110784327995, 0.6190809146227304], "top": "A", "option_logits": null, "prompt_sha256": "70f353d85ca31feb6cd6d5c421fe7a2e877a313657d44902068dc2f6ef451bf8", "input_tokens": 207, "server_ms": 34.55}], "seconds": 5.3, "failures": 0}, "multifield": {"rows": [{"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.14720923592665844, 0.8527907640733415], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.8214121479792841, 0.17115665028167817, 0.00743120173903778], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.9305404571279347, 0.06673814075012655, 0.0027214021219388934], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.8095965925134007, 0.07462670936879237, 0.11577669811780701], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.9471470189565372, 0.052852981043462664], "top": "same"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.9350414422025257, 0.05915293776423124, 0.005805620033243134], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.22187782400103293, 0.778122175998967], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.09874410354776683, 0.06776898998409085, 0.8334869064681424], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.07967467169628852, 0.9203253283037116], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.10841793826059654, 0.0843556864709907, 0.8072263752684127], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.21123545993246395, 0.788764540067536], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.11796301878836102, 0.8782943442297925, 0.0037426369818465326], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.8527907640733415, 0.14720923592665844], "top": "same"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.24350964936246988, 0.7532804299591004, 0.0032099206784296486], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.9344875966425453, 0.06294564908924384, 0.002566754268210852], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.7318060856600669, 0.2681939143399331], "top": "moved"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.727805131100519, 0.25050783114702185, 0.02168703775245924], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.8387192579214904, 0.15415437985672864, 0.007126362221781067], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.10592235395880453, 0.8940776460411956], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.928892622801621, 0.06661995929549502, 0.004487417902884053], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.9588514552888884, 0.039099387288032916, 0.002049157423078645], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.14648079183179633, 0.8485708719675852, 0.004948336200618522], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.9248060506082927, 0.07519394939170732], "top": "same"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.24334788832895243, 0.7527800186067107, 0.0038720930643368974], "top": "meet_wren"}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.8089141677995819, 0.1910858322004182], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.9439168356016766, 0.0560831643983235], "top": "same"}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.9156019658234928, 0.08439803417650704], "top": "stay"}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.3201997500236374, 0.6798002499763626], "top": "holds"}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.12511350509772579, 0.8748864949022742], "top": "departure"}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.13214384255458153, 0.8678561574454184], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.25606104458540857, 0.7439389554145914], "top": "holds"}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.08439803417650704, 0.9156019658234928], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.11840617139815589, 0.8815938286018441], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.10012611628383591, 0.8998738837161642], "top": "departure"}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.8184242551251981, 0.18157574487480185], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.12511350509772579, 0.8748864949022742], "top": "holds"}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.9248060506082927, 0.07519394939170732], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.7193152314525416, 0.2806847685474584], "top": "moved"}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.08937422170979005, 0.91062577829021], "top": "outside"}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.07519394939170732, 0.9248060506082927], "top": "outside"}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.15526082656247686, 0.844739173437523], "top": "holds"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.45308504361568214, 0.5469149563843178], "top": "holds"}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.25606104458540857, 0.7439389554145914], "top": "outside"}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.15526082656247686, 0.844739173437523], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.43758935794935333, 0.5624106420506467], "top": "outside"}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.9203253283037116, 0.07967467169628852], "top": "same"}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.936892368084393, 0.06310763191560703], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.9156019658234928, 0.08439803417650704], "top": "same"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.91062577829021, 0.08937422170979005], "top": "stay"}], "seconds": 1.0, "failures": 0}}, "finished_utc": "2026-09-30T16:28:00Z"} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/vram.csv new file mode 100644 index 0000000..0468b0b --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/vram.csv @@ -0,0 +1,824 @@ +# pid=1876154 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790785594.709607614,8812,8822 +1790785594.816992687,8812,8822 +1790785594.924898727,8812,8822 +1790785595.033046259,8812,8822 +1790785595.141369014,8812,8822 +1790785595.248597454,8812,8822 +1790785595.356106069,8812,8822 +1790785595.461684860,8812,8822 +1790785595.568335084,8812,8822 +1790785595.677359057,8812,8822 +1790785595.784828471,8812,8822 +1790785595.893056614,8812,8822 +1790785596.000834872,8812,8822 +1790785596.108520269,8812,8822 +1790785596.216519879,8812,8822 +1790785596.324879214,8812,8822 +1790785596.435465377,8812,8822 +1790785596.538976832,8812,8822 +1790785596.640537193,8812,8822 +1790785596.748622415,8812,8822 +1790785596.855910686,8812,8822 +1790785596.960802619,8812,8822 +1790785597.068066870,8812,8822 +1790785597.176499836,8812,8822 +1790785597.283548085,8812,8822 +1790785597.388865952,8812,8822 +1790785597.494897239,8812,8822 +1790785597.604805643,8812,8828 +1790785597.713374780,8818,8828 +1790785597.821776126,8818,8828 +1790785597.929445682,8818,8828 +1790785598.038907031,8818,8828 +1790785598.149278011,8818,8828 +1790785598.253166101,8818,8828 +1790785598.353414965,8818,8828 +1790785598.453979335,8818,8828 +1790785598.553564822,8818,8828 +1790785598.654889150,8818,8828 +1790785598.756197598,8818,8828 +1790785598.859435040,8818,8828 +1790785598.970913513,8818,8828 +1790785599.080827787,8818,8828 +1790785599.184755328,8818,8828 +1790785599.289914433,8818,8828 +1790785599.392404476,8818,8828 +1790785599.497476210,8818,8828 +1790785599.600688592,8818,8828 +1790785599.703101704,8818,8828 +1790785599.804872157,8818,8828 +1790785599.906468969,8818,8828 +1790785600.009585269,8818,8828 +1790785600.112002001,8818,8828 +1790785600.216424348,8818,8828 +1790785600.319480468,8818,8828 +1790785600.421649716,8818,8828 +1790785600.528636134,8834,8844 +1790785600.632067539,8834,8844 +1790785600.737269314,8834,8844 +1790785600.845817212,8834,8844 +1790785600.955924229,8834,8844 +1790785601.065167244,8834,8844 +1790785601.173459918,8834,8844 +1790785601.275585876,8834,8844 +1790785601.376744613,8834,8844 +1790785601.477399873,8834,8844 +1790785601.577554757,8834,8844 +1790785601.681883482,8834,8844 +1790785601.781832583,8834,8844 +1790785601.881254388,8834,8844 +1790785601.987499567,8834,8844 +1790785602.091864893,8834,8844 +1790785602.199146025,8834,8844 +1790785602.301492325,8834,8844 +1790785602.401721421,8834,8844 +1790785602.503718457,8834,8844 +1790785602.610046707,8834,8844 +1790785602.710030509,8834,8844 +1790785602.815224675,8834,8844 +1790785602.926878220,8834,8844 +1790785603.037626455,8834,8844 +1790785603.152249027,8834,8844 +1790785603.264584071,8834,8844 +1790785603.372698943,8834,8844 +1790785603.477801828,8834,8844 +1790785603.579362999,8834,8844 +1790785603.681387566,8834,8844 +1790785603.782639273,8834,8844 +1790785603.885451800,8834,8844 +1790785603.989455181,8834,8844 +1790785604.098090159,8834,8844 +1790785604.201761727,8834,8844 +1790785604.305011539,8834,8844 +1790785604.409081811,8834,8844 +1790785604.512517535,8834,8844 +1790785604.614802175,8834,8844 +1790785604.713588272,8834,8844 +1790785604.815137484,8834,8844 +1790785604.915937565,8834,8844 +1790785605.020434713,8834,8844 +1790785605.124779798,8834,8844 +1790785605.229297326,8834,8844 +1790785605.331117430,8834,8844 +1790785605.436974244,8834,8844 +1790785605.542899320,8834,8844 +1790785605.644078616,8834,8844 +1790785605.747651512,8834,8844 +1790785605.852402002,8834,8844 +1790785605.956299703,8834,8844 +1790785606.059734557,8834,8844 +1790785606.163191702,8834,8844 +1790785606.265541472,8834,8844 +1790785606.368260768,8834,8844 +1790785606.473078299,8834,8844 +1790785606.575602622,8834,8844 +1790785606.678994846,8834,8844 +1790785606.784679758,8834,8844 +1790785606.889345798,8834,8844 +1790785606.993861255,8834,8844 +1790785607.100494429,8834,8844 +1790785607.205149228,8834,8844 +1790785607.311194665,8834,8844 +1790785607.414642479,8834,8844 +1790785607.517948472,8834,8844 +1790785607.621723590,8834,8844 +1790785607.727616745,8834,8844 +1790785607.833617031,8834,8844 +1790785607.938896658,8834,8844 +1790785608.042700187,8834,8844 +1790785608.140533502,8834,8844 +1790785608.247990236,8834,8844 +1790785608.351987677,8834,8844 +1790785608.455081988,8834,8844 +1790785608.564665847,8834,8844 +1790785608.668684579,8834,8844 +1790785608.775319142,8834,8844 +1790785608.879589447,8834,8844 +1790785608.985530912,8834,8844 +1790785609.087797973,8834,8844 +1790785609.189815479,8834,8844 +1790785609.293544727,8834,8844 +1790785609.396092391,8834,8844 +1790785609.498586613,8834,8844 +1790785609.602181690,8834,8844 +1790785609.705227099,8834,8844 +1790785609.809577755,8834,8844 +1790785609.909819960,8834,8844 +1790785610.013946692,8834,8844 +1790785610.118052575,8834,8844 +1790785610.224352244,8834,8844 +1790785610.325897116,8834,8844 +1790785610.428928785,8834,8844 +1790785610.535526758,8834,8844 +1790785610.642880021,8834,8844 +1790785610.749195921,8834,8844 +1790785610.854299995,8834,8844 +1790785610.959345760,8834,8844 +1790785611.057644541,8834,8844 +1790785611.162914308,8834,8844 +1790785611.265354080,8834,8844 +1790785611.369277920,8834,8844 +1790785611.476475781,8834,8844 +1790785611.578745070,8834,8844 +1790785611.679811786,8834,8844 +1790785611.781109914,8834,8844 +1790785611.882821407,8834,8844 +1790785611.989731764,8834,8844 +1790785612.090828119,8834,8844 +1790785612.194495547,8834,8844 +1790785612.299318158,8854,8864 +1790785612.405717789,8854,8864 +1790785612.509670219,8854,8864 +1790785612.611490634,8854,8864 +1790785612.713851885,8854,8864 +1790785612.818102099,8854,8864 +1790785612.931263743,8854,8864 +1790785613.042444483,8854,8864 +1790785613.146651757,8854,8864 +1790785613.244824427,8854,8864 +1790785613.346989145,8854,8864 +1790785613.446492191,8854,8864 +1790785613.550027247,8854,8864 +1790785613.652009473,8854,8864 +1790785613.753048948,8854,8864 +1790785613.857993561,8854,8864 +1790785613.961286073,8854,8864 +1790785614.068251751,8854,8864 +1790785614.175608643,8854,8864 +1790785614.275942660,8854,8864 +1790785614.381020194,8854,8864 +1790785614.490005277,8854,8864 +1790785614.593374510,8854,8864 +1790785614.696966517,8854,8864 +1790785614.804012255,8854,8864 +1790785614.909376794,8854,8864 +1790785615.013724309,8854,8864 +1790785615.113710911,8854,8864 +1790785615.218621664,8854,8864 +1790785615.325510901,8854,8864 +1790785615.427057001,8854,8864 +1790785615.532458900,8854,8864 +1790785615.632612164,8854,8864 +1790785615.736959769,8854,8864 +1790785615.841860152,8854,8864 +1790785615.942890456,8854,8864 +1790785616.045516321,8854,8864 +1790785616.152524779,8854,8864 +1790785616.256817154,8854,8864 +1790785616.361177800,8854,8864 +1790785616.467057594,8854,8864 +1790785616.571422080,8854,8864 +1790785616.675638544,8854,8864 +1790785616.779704536,8854,8864 +1790785616.882084087,8854,8864 +1790785616.987007890,8854,8864 +1790785617.087598729,8854,8864 +1790785617.190860741,8854,8864 +1790785617.289878591,8854,8864 +1790785617.392270463,8854,8864 +1790785617.496012091,8854,8864 +1790785617.597668343,8854,8864 +1790785617.695188295,8854,8864 +1790785617.798671640,8854,8864 +1790785617.899695824,8854,8864 +1790785618.005153303,8854,8864 +1790785618.108155643,8854,8864 +1790785618.211467855,8854,8864 +1790785618.316917815,8854,8864 +1790785618.415973965,8854,8864 +1790785618.520995988,8854,8864 +1790785618.627517601,8854,8864 +1790785618.733268573,8854,8864 +1790785618.838960016,8854,8864 +1790785618.943700936,8854,8864 +1790785619.055231550,8854,8864 +1790785619.162038316,8854,8864 +1790785619.266056477,8854,8864 +1790785619.372013133,8854,8864 +1790785619.485965127,8854,8864 +1790785619.591275494,8854,8864 +1790785619.695828133,8854,8864 +1790785619.797798199,8854,8864 +1790785619.904368532,8854,8864 +1790785620.009285214,8854,8864 +1790785620.115275690,8854,8864 +1790785620.219060119,8854,8864 +1790785620.321849395,8854,8864 +1790785620.428324017,8854,8864 +1790785620.529572044,8854,8864 +1790785620.638067491,8854,8864 +1790785620.741757228,8854,8864 +1790785620.846063233,8854,8864 +1790785620.949969423,8854,8864 +1790785621.056030290,8854,8864 +1790785621.160429466,8854,8864 +1790785621.267196822,8854,8864 +1790785621.369844626,8854,8864 +1790785621.472031105,8854,8864 +1790785621.577285942,8854,8864 +1790785621.683730383,8854,8864 +1790785621.797087860,8854,8864 +1790785621.906694080,8854,8864 +1790785622.014675510,8854,8864 +1790785622.125799340,8854,8864 +1790785622.236610714,8854,8864 +1790785622.348098738,8854,8864 +1790785622.451529492,8854,8864 +1790785622.553111574,8854,8864 +1790785622.657991336,8854,8864 +1790785622.759256684,8854,8864 +1790785622.865012306,8854,8864 +1790785622.967288047,8854,8864 +1790785623.066208385,8854,8864 +1790785623.173430816,8854,8864 +1790785623.274718484,8854,8864 +1790785623.375495875,8854,8864 +1790785623.481760815,8854,8864 +1790785623.585582034,8854,8864 +1790785623.691741052,8854,8864 +1790785623.796050887,8854,8864 +1790785623.899095766,8854,8864 +1790785624.005883682,8854,8864 +1790785624.108590018,8854,8864 +1790785624.207309874,8854,8864 +1790785624.309983718,8854,8864 +1790785624.417641065,8854,8864 +1790785624.523136494,8854,8864 +1790785624.628142478,8854,8864 +1790785624.728287042,8854,8864 +1790785624.829741402,8854,8864 +1790785624.932206234,8854,8864 +1790785625.043059750,8854,8864 +1790785625.150292661,8854,8864 +1790785625.255469867,8854,8864 +1790785625.355052254,8854,8864 +1790785625.456987359,8854,8864 +1790785625.563916547,8854,8864 +1790785625.669842002,8854,8864 +1790785625.775950599,8854,8864 +1790785625.880499597,8854,8864 +1790785625.984476328,8854,8864 +1790785626.085764326,8854,8864 +1790785626.188315380,8854,8864 +1790785626.287356840,8854,8864 +1790785626.394873664,8854,8864 +1790785626.499558344,8854,8864 +1790785626.606170207,8854,8864 +1790785626.714858666,8854,8864 +1790785626.823257802,8854,8864 +1790785626.931415534,8854,8864 +1790785627.040469488,8854,8864 +1790785627.143434666,8854,8864 +1790785627.248763314,8854,8864 +1790785627.351460559,8854,8864 +1790785627.453001510,8854,8864 +1790785627.558009084,8854,8864 +1790785627.659347522,8854,8864 +1790785627.765135116,8854,8864 +1790785627.872018123,8854,8864 +1790785627.972452320,8854,8864 +1790785628.077579605,8854,8864 +1790785628.181635927,8854,8864 +1790785628.288393502,8854,8864 +1790785628.390023014,8854,8864 +1790785628.495781537,8854,8864 +1790785628.601113285,8854,8864 +1790785628.703923001,8854,8864 +1790785628.807478317,8854,8864 +1790785628.913873088,8854,8864 +1790785629.017692767,8854,8864 +1790785629.118337987,8854,8864 +1790785629.223488352,8854,8864 +1790785629.324346285,8854,8864 +1790785629.425777745,8854,8864 +1790785629.531257964,8854,8864 +1790785629.636408389,8854,8864 +1790785629.741149630,8854,8864 +1790785629.844872378,8854,8864 +1790785629.950318637,8854,8864 +1790785630.054754954,8854,8864 +1790785630.155855469,8854,8864 +1790785630.258610295,8854,8864 +1790785630.358727428,8854,8864 +1790785630.465868359,8854,8864 +1790785630.571165316,8854,8864 +1790785630.675700914,8854,8864 +1790785630.780877199,8854,8864 +1790785630.887339471,8854,8864 +1790785630.991424393,8854,8864 +1790785631.095633457,8854,8864 +1790785631.201364770,8854,8864 +1790785631.303251075,8854,8864 +1790785631.405481615,8854,8864 +1790785631.511842834,8854,8864 +1790785631.615997038,8854,8864 +1790785631.719476463,8854,8864 +1790785631.827154339,8854,8864 +1790785631.930559383,8854,8864 +1790785632.032668751,8854,8864 +1790785632.144925254,8854,8864 +1790785632.256684581,8854,8864 +1790785632.368422368,8854,8864 +1790785632.472093765,8854,8864 +1790785632.575447158,8854,8864 +1790785632.687123614,8854,8864 +1790785632.795828053,8854,8864 +1790785632.907902674,8854,8864 +1790785633.016265599,8854,8864 +1790785633.118044383,8854,8864 +1790785633.221721620,8854,8864 +1790785633.323794538,8854,8864 +1790785633.429383878,8854,8864 +1790785633.529978268,8854,8864 +1790785633.633481393,8854,8864 +1790785633.737002249,8854,8864 +1790785633.842145054,8854,8864 +1790785633.945815101,8854,8864 +1790785634.047307452,8854,8864 +1790785634.153913155,8854,8864 +1790785634.256097104,8854,8864 +1790785634.358168811,8854,8864 +1790785634.456825216,8854,8864 +1790785634.561727809,8854,8864 +1790785634.670975305,8854,8864 +1790785634.774198547,8854,8864 +1790785634.875493934,8854,8864 +1790785634.977562682,8854,8864 +1790785635.075459388,8854,8864 +1790785635.185832488,8854,8864 +1790785635.295771582,8854,8864 +1790785635.405594965,8854,8864 +1790785635.515770653,8854,8864 +1790785635.625450494,8854,8864 +1790785635.733832639,8854,8864 +1790785635.841840120,8854,8864 +1790785635.944543355,8854,8864 +1790785636.051086838,8854,8864 +1790785636.155643446,8854,8864 +1790785636.261064475,8854,8864 +1790785636.364041373,8854,8864 +1790785636.470979541,8854,8864 +1790785636.572649613,8854,8864 +1790785636.674608179,8854,8864 +1790785636.779137878,8854,8864 +1790785636.879718366,8854,8864 +1790785636.980323516,8854,8864 +1790785637.083720000,8854,8864 +1790785637.189940868,8854,8864 +1790785637.289978991,8854,8864 +1790785637.391302539,8854,8864 +1790785637.493199054,8854,8864 +1790785637.598102267,8854,8864 +1790785637.697995918,8854,8864 +1790785637.801950218,8854,8864 +1790785637.900464102,8854,8864 +1790785638.001100852,8854,8864 +1790785638.104380644,8854,8864 +1790785638.211928219,8854,8864 +1790785638.313021714,8854,8864 +1790785638.417312690,8854,8864 +1790785638.518800170,8854,8864 +1790785638.620347191,8854,8864 +1790785638.723609123,8854,8864 +1790785638.829371806,8854,8864 +1790785638.933789613,8854,8864 +1790785639.034823357,8854,8864 +1790785639.141157247,8854,8864 +1790785639.245816406,8854,8864 +1790785639.347584081,8854,8864 +1790785639.451502371,8854,8864 +1790785639.555872487,8854,8864 +1790785639.660972511,8854,8864 +1790785639.765607051,8854,8864 +1790785639.869784104,8854,8864 +1790785639.970669737,8854,8864 +1790785640.076277738,8854,8864 +1790785640.179462039,8854,8864 +1790785640.281087101,8854,8864 +1790785640.384766178,8854,8864 +1790785640.487522064,8854,8864 +1790785640.593664942,8854,8864 +1790785640.698969049,8854,8864 +1790785640.798103661,8854,8864 +1790785640.895792144,8854,8864 +1790785641.001206213,8854,8864 +1790785641.105196004,8854,8864 +1790785641.211726616,8854,8864 +1790785641.313044895,8854,8864 +1790785641.413627674,8854,8864 +1790785641.517159189,8854,8864 +1790785641.620146288,8854,8864 +1790785641.723693284,8854,8864 +1790785641.829768181,8854,8864 +1790785641.937343036,8854,8864 +1790785642.046708223,8854,8864 +1790785642.155712746,8854,8864 +1790785642.265418538,8854,8864 +1790785642.375133099,8854,8864 +1790785642.484202643,8854,8864 +1790785642.589452650,8854,8864 +1790785642.694025108,8854,8864 +1790785642.796358589,8854,8864 +1790785642.897145511,8854,8864 +1790785643.001402495,8854,8864 +1790785643.104206121,8854,8864 +1790785643.203781029,8854,8864 +1790785643.309912816,8854,8864 +1790785643.409834987,8854,8864 +1790785643.512463071,8854,8864 +1790785643.615878065,8854,8864 +1790785643.718539760,8854,8864 +1790785643.824241192,8854,8864 +1790785643.926702955,8854,8864 +1790785644.031460465,8854,8864 +1790785644.139667949,8854,8864 +1790785644.241470773,8854,8864 +1790785644.344773436,8854,8864 +1790785644.449656177,8854,8864 +1790785644.559841675,8854,8864 +1790785644.671732864,8854,8864 +1790785644.780489643,8854,8864 +1790785644.889532297,8854,8864 +1790785644.992899950,8854,8864 +1790785645.097801673,8854,8864 +1790785645.200466377,8854,8864 +1790785645.302586495,8854,8864 +1790785645.408118676,8854,8864 +1790785645.511590931,8854,8864 +1790785645.618421547,8854,8864 +1790785645.722858943,8854,8864 +1790785645.823196640,8854,8864 +1790785645.924369596,8854,8864 +1790785646.026633616,8854,8864 +1790785646.127737251,8854,8864 +1790785646.231319237,8854,8864 +1790785646.332045189,8854,8864 +1790785646.432101101,8854,8864 +1790785646.533951646,8854,8864 +1790785646.639285664,8854,8864 +1790785646.738037250,8854,8864 +1790785646.841626186,8854,8864 +1790785646.943460241,8854,8864 +1790785647.045925603,8854,8864 +1790785647.149280537,8854,8864 +1790785647.251772589,8854,8864 +1790785647.351781811,8854,8864 +1790785647.454144143,8854,8864 +1790785647.555127807,8854,8864 +1790785647.662187066,8854,8864 +1790785647.765850033,8854,8864 +1790785647.865377349,8854,8864 +1790785647.971569857,8854,8864 +1790785648.075686310,8854,8864 +1790785648.175992786,8854,8864 +1790785648.277934072,8854,8864 +1790785648.383741525,8854,8864 +1790785648.487893378,8854,8864 +1790785648.594739084,8854,8864 +1790785648.696327196,8854,8864 +1790785648.795881343,8854,8864 +1790785648.899611720,8854,8864 +1790785649.003251587,8854,8864 +1790785649.101637009,8854,8864 +1790785649.200916352,8854,8864 +1790785649.303778150,8854,8864 +1790785649.412129084,8854,8864 +1790785649.519651299,8854,8864 +1790785649.621185050,8854,8864 +1790785649.725719988,8854,8864 +1790785649.827595463,8854,8864 +1790785649.933179354,8854,8864 +1790785650.045646669,8854,8864 +1790785650.155756256,8854,8864 +1790785650.259020279,8854,8864 +1790785650.356623231,8854,8864 +1790785650.457543284,8854,8864 +1790785650.559331198,8854,8864 +1790785650.661498247,8854,8864 +1790785650.771961958,8854,8864 +1790785650.872335334,8854,8864 +1790785650.974646645,8854,8864 +1790785651.077901647,8854,8864 +1790785651.177697246,8854,8864 +1790785651.282428017,8854,8864 +1790785651.385221513,8854,8864 +1790785651.489774321,8854,8864 +1790785651.594009955,8854,8864 +1790785651.697537431,8854,8864 +1790785651.800763343,8854,8864 +1790785651.911732499,8854,8864 +1790785652.024282926,8854,8864 +1790785652.133680344,8854,8864 +1790785652.241492892,8854,8864 +1790785652.343251496,8854,8864 +1790785652.446881842,8854,8864 +1790785652.552381663,8854,8864 +1790785652.655473402,8854,8864 +1790785652.760950592,8854,8864 +1790785652.866045897,8854,8864 +1790785652.967181823,8854,8864 +1790785653.071968044,8854,8864 +1790785653.184636942,8854,8864 +1790785653.291100433,8854,8864 +1790785653.397361913,8854,8864 +1790785653.501204822,8854,8864 +1790785653.605776440,8854,8864 +1790785653.714114605,8854,8864 +1790785653.815012668,8854,8864 +1790785653.917164536,8854,8864 +1790785654.015434837,8854,8864 +1790785654.117825099,8854,8864 +1790785654.216878069,8854,8864 +1790785654.317931454,8854,8864 +1790785654.421765823,8854,8864 +1790785654.523125302,8854,8864 +1790785654.628147816,8854,8864 +1790785654.731920454,8854,8864 +1790785654.837315193,8854,8864 +1790785654.941397685,8854,8864 +1790785655.042041845,8854,8864 +1790785655.144354195,8854,8864 +1790785655.252504657,8854,8864 +1790785655.352753963,8854,8864 +1790785655.453542924,8854,8864 +1790785655.559276537,8854,8864 +1790785655.661613058,8854,8864 +1790785655.761499268,8854,8864 +1790785655.862008367,8854,8864 +1790785655.963188953,8854,8864 +1790785656.065701606,8854,8864 +1790785656.167420429,8854,8864 +1790785656.265883093,8854,8864 +1790785656.365999946,8854,8864 +1790785656.470636215,8854,8864 +1790785656.571983484,8854,8864 +1790785656.675157864,8854,8864 +1790785656.770076344,8854,8864 +1790785656.882040773,8854,8864 +1790785656.986582581,8854,8864 +1790785657.088096322,8854,8864 +1790785657.194867777,8854,8864 +1790785657.295391986,8854,8864 +1790785657.401258700,8854,8864 +1790785657.507727692,8854,8864 +1790785657.610978234,8854,8864 +1790785657.721535666,8854,8864 +1790785657.829407275,8854,8864 +1790785657.937460186,8854,8864 +1790785658.045368266,8854,8864 +1790785658.152316653,8854,8864 +1790785658.257173885,8854,8864 +1790785658.362432112,8854,8864 +1790785658.462744758,8854,8864 +1790785658.566826580,8854,8864 +1790785658.670335115,8854,8864 +1790785658.774224655,8854,8864 +1790785658.881050261,8854,8864 +1790785658.984320753,8854,8864 +1790785659.086646194,8854,8864 +1790785659.186878329,8854,8864 +1790785659.289306231,8854,8864 +1790785659.393185590,8854,8864 +1790785659.498269055,8854,8864 +1790785659.601874641,8854,8864 +1790785659.700954012,8854,8864 +1790785659.807407924,8854,8864 +1790785659.913455170,8854,8864 +1790785660.017391620,8854,8864 +1790785660.118060221,8854,8864 +1790785660.220657345,8854,8864 +1790785660.325593918,8854,8864 +1790785660.425862153,8854,8864 +1790785660.530426331,8854,8864 +1790785660.629716874,8854,8864 +1790785660.731678281,8854,8864 +1790785660.837348882,8854,8864 +1790785660.935118727,8854,8864 +1790785661.033189635,8854,8864 +1790785661.135555216,8854,8864 +1790785661.238879089,8854,8864 +1790785661.343719241,8854,8864 +1790785661.446955742,8854,8864 +1790785661.549472226,8854,8864 +1790785661.647981909,8854,8864 +1790785661.748712160,8854,8864 +1790785661.850853499,8854,8864 +1790785661.949561955,8854,8864 +1790785662.054092783,8854,8864 +1790785662.157320975,8854,8864 +1790785662.257030553,8854,8864 +1790785662.360456107,8854,8864 +1790785662.466627265,8854,8864 +1790785662.570713997,8854,8864 +1790785662.673448843,8854,8864 +1790785662.778009681,8854,8864 +1790785662.883755884,8854,8864 +1790785662.985665919,8854,8864 +1790785663.086874696,8854,8864 +1790785663.189847025,8854,8864 +1790785663.289930788,8854,8864 +1790785663.393117800,8854,8864 +1790785663.500113727,8854,8864 +1790785663.604788197,8854,8864 +1790785663.707929148,8854,8864 +1790785663.813982805,8854,8864 +1790785663.918029046,8854,8864 +1790785664.020136064,8854,8864 +1790785664.125802256,8854,8864 +1790785664.227781392,8854,8864 +1790785664.332187569,8854,8864 +1790785664.433616368,8854,8864 +1790785664.537557579,8854,8864 +1790785664.643919939,8854,8864 +1790785664.748903963,8854,8864 +1790785664.851103862,8854,8864 +1790785664.948534932,8854,8864 +1790785665.053439235,8854,8864 +1790785665.153960233,8854,8864 +1790785665.255233410,8854,8864 +1790785665.360486567,8854,8864 +1790785665.463197322,8854,8864 +1790785665.561966009,8854,8864 +1790785665.664558163,8854,8864 +1790785665.768660126,8854,8864 +1790785665.870709893,8854,8864 +1790785665.976617858,8854,8864 +1790785666.081083105,8854,8864 +1790785666.184922844,8854,8864 +1790785666.290128080,8854,8864 +1790785666.396519021,8854,8864 +1790785666.501864469,8854,8864 +1790785666.604225240,8854,8864 +1790785666.711724464,8854,8864 +1790785666.816999921,8854,8864 +1790785666.921775622,8854,8864 +1790785667.024338936,8854,8864 +1790785667.129381180,8854,8864 +1790785667.233069307,8854,8864 +1790785667.334801310,8854,8864 +1790785667.436224840,8854,8864 +1790785667.539032217,8854,8864 +1790785667.641230855,8854,8864 +1790785667.743998032,8854,8864 +1790785667.846042019,8854,8864 +1790785667.956931335,8854,8864 +1790785668.070053719,8854,8864 +1790785668.179762881,8854,8864 +1790785668.293964637,8854,8864 +1790785668.402231351,8854,8864 +1790785668.513072687,8854,8864 +1790785668.622077640,8854,8864 +1790785668.731545128,8854,8864 +1790785668.841303681,8854,8864 +1790785668.952624542,8854,8864 +1790785669.065355691,8854,8864 +1790785669.171219395,8854,8864 +1790785669.280950577,8854,8864 +1790785669.381254763,8854,8864 +1790785669.483037746,8854,8864 +1790785669.589780442,8854,8864 +1790785669.690828536,8854,8864 +1790785669.796899994,8854,8864 +1790785669.897314940,8854,8864 +1790785670.000112937,8854,8864 +1790785670.112009575,8854,8864 +1790785670.214147534,8854,8864 +1790785670.319342370,8854,8864 +1790785670.417605550,8854,8864 +1790785670.519093051,8854,8864 +1790785670.632615840,8854,8864 +1790785670.743244022,8854,8864 +1790785670.851329764,8854,8864 +1790785670.959964332,8854,8864 +1790785671.060960067,8854,8864 +1790785671.161829709,8854,8864 +1790785671.261133462,8854,8864 +1790785671.361059394,8854,8864 +1790785671.460993665,8854,8864 +1790785671.571654898,8854,8864 +1790785671.681569132,8854,8864 +1790785671.790777628,8854,8864 +1790785671.896645112,8854,8864 +1790785672.005109968,8854,8864 +1790785672.113298191,8854,8864 +1790785672.221216850,8854,8864 +1790785672.324709416,8854,8864 +1790785672.425505127,8854,8864 +1790785672.529875124,8854,8864 +1790785672.633702922,8854,8864 +1790785672.734297232,8854,8864 +1790785672.834032011,8854,8864 +1790785672.935139576,8854,8864 +1790785673.039408321,8854,8864 +1790785673.140187383,8854,8864 +1790785673.242156959,8854,8864 +1790785673.344264167,8854,8864 +1790785673.449675695,8854,8864 +1790785673.551592681,8854,8864 +1790785673.657649178,8854,8864 +1790785673.762956685,8854,8864 +1790785673.863006598,8854,8864 +1790785673.963875760,8854,8864 +1790785674.068041434,8854,8864 +1790785674.170833600,8854,8864 +1790785674.272508492,8854,8864 +1790785674.375554602,8854,8864 +1790785674.477973344,8854,8864 +1790785674.580528617,8854,8864 +1790785674.682939619,8854,8864 +1790785674.783727361,8854,8864 +1790785674.886254854,8854,8864 +1790785674.987336729,8854,8864 +1790785675.088155371,8854,8864 +1790785675.192890192,8854,8864 +1790785675.295780289,8854,8864 +1790785675.398717147,8854,8864 +1790785675.501804587,8854,8864 +1790785675.603838164,8854,8864 +1790785675.708231611,8854,8864 +1790785675.813729530,8854,8864 +1790785675.914887476,8854,8864 +1790785676.015763339,8854,8864 +1790785676.119073202,8854,8864 +1790785676.223206535,8854,8864 +1790785676.323034805,8854,8864 +1790785676.425758550,8854,8864 +1790785676.531007657,8854,8864 +1790785676.636977882,8854,8864 +1790785676.739595417,8854,8864 +1790785676.837255100,8854,8864 +1790785676.940134217,8854,8864 +1790785677.042175315,8854,8864 +1790785677.145189514,8854,8864 +1790785677.247964760,8854,8864 +1790785677.349731944,8854,8864 +1790785677.451074872,8854,8864 +1790785677.550168413,8854,8864 +1790785677.656059798,8854,8864 +1790785677.759163548,8854,8864 +1790785677.864496055,8854,8864 +1790785677.965846244,8854,8864 +1790785678.071502246,8854,8864 +1790785678.173553063,8854,8864 +1790785678.279044753,8854,8864 +1790785678.380159408,8854,8864 +1790785678.489429105,8854,8864 +1790785678.591152598,8854,8864 +1790785678.695319761,8854,8864 +1790785678.799138590,8854,8864 +1790785678.901463131,8854,8864 +1790785679.005562223,8854,8864 +1790785679.110572877,8854,8864 +1790785679.212006447,8854,8864 +1790785679.315733905,8854,8864 +1790785679.420062610,8854,8864 +1790785679.528223462,8928,8938 +1790785679.635479994,8928,8938 +1790785679.742866637,8928,8938 +1790785679.845526532,8928,8938 +1790785679.947059622,8928,8938 +1790785680.047562120,8928,8938 +1790785680.149316124,8928,8938 +1790785680.252140871,8928,8938 +1790785680.356649089,8928,8938 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/container.id b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/container.id new file mode 100644 index 0000000..99a98e8 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/container.id @@ -0,0 +1 @@ +d10e0ec2b1f031f57d6594c10548696bd8352c75b35458cf44538ead4fc2eb0a diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/events.txt b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/events.txt new file mode 100644 index 0000000..f1cfbb4 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/events.txt @@ -0,0 +1,3 @@ +1790785726.691427913 sets:start +1790785810.443650874 sets:done +gpu3 after: 2 MiB diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/health-end.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/health-end.json new file mode 100644 index 0000000..7fb2695 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/health-end.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":8.082,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/health-rest.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/health-rest.json new file mode 100644 index 0000000..37aac99 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/health-rest.json @@ -0,0 +1 @@ +{"status":"ok","model":{"name":"Intern-Decision-4B","source":"internlm/Intern-Decision-4B","revision":"0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","checkpoint":"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd","inference_py_sha256":"c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863","temperature":1.99241824,"dtype":"bfloat16","attn_implementation":"sdpa","device":"cuda","max_length":8192,"torch_version":"2.10.0+cu128","transformers_version":"5.17.0","vision_tower":"removed","device_name":"NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition","allocated_gib":7.937,"reserved_gib":7.969,"max_reserved_gib":8.861},"vram_cap_gib":9.25,"max_tokens":8192,"max_decisions":64,"max_questions_per_call":16,"chunking":"/decide/shared questions are packed greedily, in request order, into calls of at most 16 (1-16, 17-32, ...); each call is one prompt, so the questions in a call are asked together. With orderings, ordering k of every decision forms wave k, packed the same way.","workloads":[]} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/sets.json b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/sets.json new file mode 100644 index 0000000..e9da891 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/sets.json @@ -0,0 +1 @@ +{"label": "r3", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:28:46Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.864141857251525, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 44.21}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.63944788463414, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 35.78}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72842790931463, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 35.03}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90549002587795, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 35.22}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.106231950223446, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 35.42}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93953103013337, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 35.25}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76771808043122, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 35.03}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.932720966637135, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 35.22}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85064979456365, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.71}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46167500503361, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.64}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.240603007376194, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.38}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.52531802840531, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 35.73}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78184880502522, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.88}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.001950200647116, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.2}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13770180568099, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 35.21}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60254709050059, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.92}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.83117979764938, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 35.12}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68751700408757, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 35.05}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.727637914940715, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.9}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7791291102767, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 35.15}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.80177910812199, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 35.09}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.96608094125986, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 35.04}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94414992444217, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 35.27}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85123908892274, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 35.16}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8628799226135, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 35.11}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.928291130810976, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 35.29}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82443888485432, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 35.06}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72959788329899, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 35.12}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48044604435563, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.79}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01415208727121, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.25}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62974706478417, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.93}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63482812792063, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.93}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45043510571122, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.72}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65220707282424, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.98}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57199598476291, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.86}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13018184155226, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 35.32}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.754627875983715, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 35.0}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.704397924244404, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 35.05}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78058895841241, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 35.14}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72836807183921, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.95}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72195801138878, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.97}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.502966122701764, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.75}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43828507140279, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.8}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.388084853067994, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 35.69}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.159431958571076, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 34.44}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43851510807872, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.68}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44743391871452, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.67}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.857169983908534, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 35.0}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.89494203031063, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 36.05}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76838909834623, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.86}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.364644860848784, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 35.43}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90892008692026, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 35.04}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.093950901180506, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.55}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67618793062866, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.96}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54969616420567, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.65}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.541726138442755, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.76}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31711395084858, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.55}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.250762805342674, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.61}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.194132804870605, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.51}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.707517854869366, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.94}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1264920104295, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.51}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42610407061875, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.76}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.4825819209218, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 39.8}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55279607139528, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.73}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.028279853984714, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.4}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1424809563905, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.46}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03495999798179, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.35}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48946497216821, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.73}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.964328959584236, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.31}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24590293131769, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.56}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27898304350674, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.61}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.365384167060256, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.65}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28935192339122, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.46}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21916303038597, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.58}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01762994751334, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.3}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92769997380674, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 35.01}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.47115915454924, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 37.8}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49065496958792, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.7}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.691316800192, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.93}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73684799671173, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.98}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31424291618168, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.47}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46916516683996, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.75}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15404192917049, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.46}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57919720374048, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.76}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.458714097738266, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.69}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43068501353264, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.74}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.234912160784006, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.61}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69433801062405, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 35.02}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38091397099197, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.73}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45748498477042, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.74}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.303073797374964, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.53}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65096701681614, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 35.0}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20138198696077, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.5}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47172490693629, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.66}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13787207379937, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.52}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24716207943857, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.56}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.289963103830814, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.53}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36521420069039, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.62}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07021987065673, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.39}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54014614969492, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.78}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30210312642157, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.57}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3614439740777, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.6}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96725005283952, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.36}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27072197943926, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.58}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31470322236419, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.52}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52595582045615, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.9}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.019639974460006, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.31}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61149710044265, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.83}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.012469954788685, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.37}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31681313365698, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.61}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09140093997121, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.32}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20348188653588, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.55}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71356795914471, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 35.02}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.97872085124254, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 35.14}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72279806435108, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.87}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.847688887268305, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 35.06}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.388543831184506, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.61}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.494105054065585, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.77}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50352608039975, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.88}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66015684232116, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 35.05}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.378064123913646, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.87}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64242702163756, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 35.13}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1442218311131, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.67}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61334693990648, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 35.15}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.482666015625, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 35.86}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.02795195952058, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 35.41}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5190160702914, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.98}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.773708114400506, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 35.03}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24908200092614, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.62}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70949901081622, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 35.13}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49485616572201, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.82}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.96054096892476, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 35.38}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56650583818555, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.89}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35704394057393, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.77}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40515396930277, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.81}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4350450001657, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.79}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5951670575887, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.9}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.731877993792295, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 35.12}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34495397657156, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.69}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.499574944376945, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.94}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35522413626313, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.76}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48803515918553, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.92}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40538493730128, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.69}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44432493112981, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.81}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32600389048457, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.73}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81253891065717, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 35.21}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38315403461456, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.78}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61220700107515, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.93}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.167852045968175, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.56}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74763783439994, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 35.21}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62140707112849, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 34.94}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.338703870773315, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.75}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26069186814129, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 34.65}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59828698821366, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 35.0}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39341488108039, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.81}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.95857005752623, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 35.26}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.416934967041016, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.79}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51385514624417, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.9}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41082502529025, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 34.78}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.589346058666706, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.92}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3958139680326, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 34.76}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.95376094058156, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 35.33}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.372083868831396, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 34.69}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67550806514919, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 35.1}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49135509133339, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 34.83}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8192791249603, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 35.17}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48776484280825, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 34.92}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6036970410496, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.95}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.257043179124594, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.68}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.710357923060656, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 35.14}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50102501362562, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 34.87}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46412508189678, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.95}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20538201741874, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 34.63}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6378670334816, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 35.09}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44194516912103, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 34.9}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52515581436455, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.96}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45584492571652, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 34.92}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53176610916853, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 35.01}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.100380985066295, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 34.6}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63687703758478, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 35.13}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.373204017058015, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 34.7}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.32925013639033, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 39.76}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.112447088584304, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 39.3}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.9007821213454, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 36.13}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.176520796492696, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 34.65}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.704108050093055, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 35.22}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35492401756346, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.7}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.06194197386503, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 35.45}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46325499378145, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 34.81}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7703991252929, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 35.19}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29581297375262, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 34.82}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4566949903965, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.94}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33797394670546, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 34.75}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5413759611547, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 35.06}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04984998330474, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 34.6}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56289603188634, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 35.09}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33702399767935, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 34.87}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50742506235838, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 35.05}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31605410575867, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 34.86}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50528511591256, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 35.0}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12757187895477, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.64}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85319905728102, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 35.19}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.323643919080496, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 34.84}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.44634583219886, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 35.95}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09308081306517, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 34.65}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38147392868996, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.94}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.070841062814, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 34.6}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47742506675422, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 35.01}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11087116971612, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 34.66}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.001541048288345, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 35.51}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39858511835337, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.81}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70985780283809, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 35.09}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.825139006599784, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 35.23}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52862592227757, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 35.04}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31930302456021, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 34.88}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65487694181502, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 35.19}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24394296295941, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.73}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.731948075816035, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 35.26}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.294793874025345, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 34.88}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75129816494882, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 35.33}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.148102885112166, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 34.61}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74218703620136, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 35.2}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2726630885154, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.83}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41087601520121, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.96}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.695566890761256, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 35.21}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.664016941562295, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 35.14}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02155011706054, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 34.53}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.871198961511254, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 35.42}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14810195378959, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 34.73}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42839605547488, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.96}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56478615105152, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 35.1}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31351182796061, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.83}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22900291904807, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 34.74}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.930371006950736, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 35.32}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.200351944193244, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 34.67}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47521503828466, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 35.01}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6573670655489, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 35.03}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.806618958711624, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 35.07}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25026291795075, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 34.81}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.674276856705546, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 35.18}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.51466295681894, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 40.0}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08207204565406, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 35.56}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366334116086364, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 34.87}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71233814768493, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 35.24}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23176312632859, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 34.75}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77979803271592, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 35.3}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4995949678123, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 35.03}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4454150442034, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 34.97}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.566745886579156, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 35.03}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53646602854133, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 35.03}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10926114581525, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 34.67}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.707638926804066, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 35.18}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.72917895205319, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 36.1}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.361964117735624, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.84}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74435794912279, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 35.32}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32239385135472, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.87}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60559717193246, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 35.07}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.716018967330456, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.11}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35892288386822, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 34.86}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.499005906283855, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.92}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.791520124301314, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 35.25}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43220493197441, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.93}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76335809379816, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 35.26}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33093407936394, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.82}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.328673992305994, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.81}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48705401085317, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.93}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.585776997730136, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 35.05}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52361610531807, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.97}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47016494162381, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 34.96}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30613295920193, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.81}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92472090572119, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 35.42}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1786009054631, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.72}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54475596360862, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3548639472574, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.87}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.473185919225216, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 34.82}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.624737014994025, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.06}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64709713868797, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 35.03}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42145420983434, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.88}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79520886018872, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 35.28}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44866489432752, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.96}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75623789988458, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 35.21}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65547787770629, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.94}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81474893912673, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 35.2}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51569497399032, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.01}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.477536126971245, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 34.97}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55337605252862, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.04}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.838330026715994, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 35.38}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33998304232955, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.85}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51085605472326, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69141715019941, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.08}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71794796921313, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 35.17}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57282695733011, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.0}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81500901095569, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 35.25}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35867398791015, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.83}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.840599892660975, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 35.38}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32176311314106, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.84}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56334609352052, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 35.06}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.413384998217225, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.91}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.85209290497005, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 37.09}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.41310802474618, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.42}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76982915587723, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 35.16}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43530483916402, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.93}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63874713331461, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 35.12}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38515404984355, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.8}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.487235153093934, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58123703300953, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.08}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.823888938874006, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 35.35}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31913398951292, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.82}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51540603116155, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 35.03}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37758402712643, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.87}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.629356840625405, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 35.16}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31078305281699, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.79}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56578592583537, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 35.09}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.616857094690204, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.06}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.575777154415846, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50291489996016, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.88}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.756348960101604, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 35.14}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.237801821902394, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.74}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85783005878329, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 35.27}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54756599664688, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.9}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39679418504238, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 34.86}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31870301999152, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58099698461592, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 35.09}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.869049003347754, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.38}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.373694125562906, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38714405335486, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.89}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29439293965697, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 34.8}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31126291677356, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.85}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.304844008758664, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 34.79}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42793411947787, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.94}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.247472908347845, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 34.8}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30977303348482, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.79}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34334395267069, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 34.88}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37955400720239, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60080598294735, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 35.14}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38803509622812, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.94}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13769116438925, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 34.66}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48508612625301, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29576291516423, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.393674021586776, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56937594152987, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 35.06}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.462905049324036, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.0}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33351407572627, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 34.73}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64632693491876, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.04}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84284996800125, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 35.22}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72425698116422, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.07}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35849400982261, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44993498548865, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.88}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40213508717716, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 34.79}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39743400365114, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.74}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40336489863694, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.93}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30588289909065, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.81}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30693296529353, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 34.8}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.472425028681755, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.94}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17757202684879, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 34.74}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39811386913061, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.89}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.436275044456124, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33840398490429, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98298893868923, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 34.33}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.97671105712652, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.09}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64779702760279, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 34.83}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.544266087934375, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.72}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.787438835948706, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 34.99}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51504607498646, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7739779651165, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 35.06}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73726802319288, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.06}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57518599741161, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 34.89}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6751481089741, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.99}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7170080319047, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.850369000807405, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.13}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66121715307236, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 35.0}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.00528100691736, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.3}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70067682489753, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72856784053147, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.673037964850664, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67233807593584, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.97}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.773658987134695, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 35.12}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84997994080186, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.07}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58868598379195, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 34.94}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98744105733931, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.23}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23948309198022, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 34.49}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39311499521136, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.59}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55412600003183, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 34.86}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.781617837026715, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.12}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48115501366556, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 34.75}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7711090259254, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.11}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.692817859351635, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 34.95}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.91426005773246, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.17}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82517905347049, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 35.11}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70971694029868, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79087890870869, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 34.98}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5682960730046, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.72}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.60669902153313, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 36.9}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.97744098119438, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.27}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.10820302553475, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 36.36}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86978009156883, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5784269310534, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 35.86}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67104819417, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 35.02}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.99056185036898, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 36.26}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.974521052092314, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.2}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5203470941633, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 35.77}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92876098304987, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.23}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.57502806931734, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 35.79}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82792994752526, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.04}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23308297432959, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 35.53}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68093721009791, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.98}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.687628926709294, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 35.99}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85830014385283, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.14}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59728807583451, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 35.86}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.95829112641513, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.13}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.389254827052355, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 35.55}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.741828011348844, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.07}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.643958047032356, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 35.9}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55591600202024, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.9}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.91511191613972, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 36.2}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01700100302696, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.29}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.67675899527967, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 35.99}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.969970842823386, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.3}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.217425044626, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 36.47}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.192453095689416, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.44}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38098607957363, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 35.68}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56592585705221, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.01199297979474, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 36.13}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.89466992020607, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.02}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.50993700325489, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 35.84}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.002570858225226, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.22}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.614647928625345, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 35.86}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55496582016349, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.68}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.26855402439833, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 35.54}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86661010235548, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 35.1}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.78137096762657, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 36.06}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.89620008319616, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.24}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.67007908411324, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 35.86}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24783193320036, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.49}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59282797202468, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45309496298432, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.71}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.29508484341204, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 35.37}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50916593708098, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.78}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.92631107009947, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 36.12}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.804699037224054, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.08}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.45173609443009, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 35.8}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65185712836683, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.98}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.57613797299564, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 35.83}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15093310549855, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.36}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32554491236806, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 35.59}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61526616290212, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.9}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.863840978592634, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 36.1}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69906796514988, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.8}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.76219899393618, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.97}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92172996327281, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.25}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.91496197320521, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 36.05}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.869758903980255, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.09}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35664503276348, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 35.61}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82410896383226, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 35.04}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.74145890586078, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 35.97}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52604592591524, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.82}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69960899464786, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 35.98}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.854179041460156, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.12}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.97051210515201, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 36.17}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.916050059720874, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 35.19}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.27577410265803, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.48}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32923301868141, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.58}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.346175104379654, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 35.6}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.803299026563764, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.04}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.52124698460102, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 35.53}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.702156929299235, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.0}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59377815388143, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 35.62}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51058503985405, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.63}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93741995468736, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 35.04}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51292615197599, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.55}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.47159584984183, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 35.68}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77266922406852, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.05}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.89862191863358, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 36.11}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.97840094193816, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.21}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.510436890646815, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 35.71}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61100699193776, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.83}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38943596929312, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 35.68}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.9563111346215, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.21}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.728089071810246, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 35.78}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70864801481366, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.91}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.70959999039769, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 35.98}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.832668982446194, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.17}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.460946874693036, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.7}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73726802319288, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 35.0}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.60107809305191, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 35.84}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.674706894904375, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.01}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.34930504672229, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 35.62}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64670714735985, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.98}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.66217890754342, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.8}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73190802894533, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.02}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.73983887024224, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 35.93}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.809259025380015, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 35.0}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.471455918625, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.76}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78899903409183, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.0}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.355065044015646, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67991717904806, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.91}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.46189705654979, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 35.66}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.886910839006305, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.2}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.4440659545362, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 35.7}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.531535839661956, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.83}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.193132961168885, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 35.35}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82033012248576, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.1}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38132577948272, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 35.57}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73838793858886, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.96}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.55607705190778, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 35.84}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78413916006684, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.05}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6820980347693, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 35.91}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74134907871485, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.98}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.34989620372653, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.59}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82934905029833, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.11}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.71362004429102, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.97}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.80588903464377, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.06}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51228696294129, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 35.74}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90592006407678, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.22}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.47338701598346, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 35.74}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.908959900960326, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.14}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.20360382087529, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79734903946519, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.406135980039835, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 35.67}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75116894394159, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.05}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.50652603432536, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 35.73}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.960100919008255, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.1}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8672000952065, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.03}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68922807648778, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.01}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15445210784674, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66348818317056, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.98}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.66135817952454, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 35.9}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77297888882458, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.97}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.2765440735966, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 35.48}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.64666396193206, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 39.17}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.51468204893172, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 39.15}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.150873897597194, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 36.21}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.36906694434583, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 36.44}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.156503112986684, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.33}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.82409096695483, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 36.29}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.285624934360385, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.121853886172175, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 36.17}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28741516917944, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.45843602903187, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 35.53}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.811218993738294, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28051406890154, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 35.41}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.11487299390137, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.28}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.2318131159991, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 35.42}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.89237001910806, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.66582899354398, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 35.86}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.16916421055794, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.672938011586666, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 35.94}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.590377032756805, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.79}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.619837956503034, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 35.89}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94849989935756, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.1}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.44582592137158, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 35.71}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.671998143196106, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.8}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.262044079601765, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 35.56}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94076107442379, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.17}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.29099517315626, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 35.49}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.480385042726994, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.25576486997306, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 35.58}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50865603797138, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.79}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38125499896705, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.68}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63612709008157, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.87}], "seconds": 21.3, "failures": 0}, "rotations": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.07268514670432, "probs": [0.021278230729761302, 0.041549605758459535, 0.9371721635117792], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.13141597807407, "probs": [0.9569751557868662, 0.022187022115204392, 0.020837822097929372], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.85347206890583, "probs": [0.028764030986406904, 0.9453276881980787, 0.025908280815514436], "top": "supported", "agreement": 1.0, "input_tokens": 196}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.72193090431392, "probs": [0.9253638152569406, 0.02190751448970375, 0.052728670253355744], "top": "insufficient", "agreement": 1.0, "input_tokens": 196}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.30311584100127, "probs": [0.9597742641946788, 0.021791404944545737, 0.01843433086077556], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.67897085659206, "probs": [0.0761485956363141, 0.025667636008847486, 0.8981837683548385], "top": "contradicted", "agreement": 1.0, "input_tokens": 206}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.38641705736518, "probs": [0.9159676813214028, 0.0555726559991739, 0.028459662679423257], "top": "contradicted", "agreement": 1.0, "input_tokens": 208}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.84210201725364, "probs": [0.02148731809580675, 0.05171731055352475, 0.9267953713506685], "top": "insufficient", "agreement": 1.0, "input_tokens": 200}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.13140503503382, "probs": [0.025695874936561842, 0.016912927133118746, 0.9573911979303193], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.89517296850681, "probs": [0.9650011484904946, 0.014421131045601873, 0.02057772046390364], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.07554500550032, "probs": [0.9685134503822489, 0.011261347946432, 0.020225201671319158], "top": "supported", "agreement": 1.0, "input_tokens": 193}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.07334701530635, "probs": [0.015601263068851802, 0.9601947660643307, 0.024203970866817515], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.7269709892571, "probs": [0.008885594588144235, 0.029266695521859745, 0.9618477098899961], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.28788794949651, "probs": [0.0467350628868348, 0.02343846611289458, 0.9298264710002706], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.3053979780525, "probs": [0.02661110718913202, 0.02251153411335893, 0.950877358697509], "top": "contradicted", "agreement": 1.0, "input_tokens": 199}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.31524787843227, "probs": [0.9629993612353499, 0.018113485593456775, 0.01888715317119321], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.66106192022562, "probs": [0.8820069055961203, 0.07635736188763995, 0.04163573251623969], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.92812316119671, "probs": [0.02701681560472035, 0.9453954270569487, 0.027587757338330996], "top": "supported", "agreement": 1.0, "input_tokens": 198}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.47815878130496, "probs": [0.02293683426823875, 0.9487912139983355, 0.028271951733425657], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.19562601298094, "probs": [0.019367602097265263, 0.013573064959569787, 0.9670593329431649], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57744901254773, "probs": [0.03987925527749218, 0.02220468207692111, 0.9379160626455868], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55095893889666, "probs": [0.02674495094579214, 0.9358820879397224, 0.03737296111448533], "top": "contradicted", "agreement": 1.0, "input_tokens": 211}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.41070914082229, "probs": [0.03051840202788244, 0.9419931233693989, 0.027488474602718756], "top": "contradicted", "agreement": 1.0, "input_tokens": 201}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.6555199213326, "probs": [0.9573434428870992, 0.022664616232544885, 0.019991940880355972], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.98945401236415, "probs": [0.20778523836492876, 0.6164398690397396, 0.17577489259533158], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76310211420059, "probs": [0.01494019219967109, 0.958782768824845, 0.026277038975483776], "top": "permitted", "agreement": 1.0, "input_tokens": 218}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.28965699672699, "probs": [0.038717451842913995, 0.06953592628537188, 0.8917466218717142], "top": "permitted", "agreement": 1.0, "input_tokens": 203}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.77618207409978, "probs": [0.8769652162510786, 0.06982844563770668, 0.053206338111214745], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.0658948738128, "probs": [0.8731504724710493, 0.09124479732837286, 0.03560473020057783], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.02813393436372, "probs": [0.03149783270455631, 0.9131032971636024, 0.055398870131841214], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.54077888466418, "probs": [0.1973515606103041, 0.4555434753670417, 0.3471049640226543], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 209}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.53607896529138, "probs": [0.02274552846853418, 0.9023369803119639, 0.07491749121950192], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.64276988245547, "probs": [0.09302877372922651, 0.07088401170776323, 0.8360872145630103], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.64905002340674, "probs": [0.8635450750116728, 0.09024102515984218, 0.04621389982848493], "top": "permitted", "agreement": 1.0, "input_tokens": 211}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.28471702896059, "probs": [0.028891977481104857, 0.0412263798327744, 0.9298816426861207], "top": "permitted", "agreement": 1.0, "input_tokens": 202}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.41496901027858, "probs": [0.8174524855587436, 0.08365642576901129, 0.098891088672245], "top": "insufficient", "agreement": 1.0, "input_tokens": 204}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.74870193563402, "probs": [0.017660819839882137, 0.04340570512192549, 0.9389334750381924], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.40336892195046, "probs": [0.06132539226815876, 0.8373735739130129, 0.10130103381882835], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3684169203043, "probs": [0.887606329608409, 0.06778068214997746, 0.04461298824161357], "top": "prohibited", "agreement": 1.0, "input_tokens": 213}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.22899693064392, "probs": [0.6913075067568434, 0.07074700515018907, 0.23794548809296753], "top": "insufficient", "agreement": 1.0, "input_tokens": 212}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.16593591496348, "probs": [0.9055821252398959, 0.059736224410951576, 0.03468165034915258], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.2951580863446, "probs": [0.034796330955504114, 0.9473839012637663, 0.017819767780729617], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.78563197143376, "probs": [0.06676731835885638, 0.04041944853804973, 0.8928132331030938], "top": "permitted", "agreement": 1.0, "input_tokens": 208}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.42512019351125, "probs": [0.0792938381036179, 0.09571489465194817, 0.8249912672444338], "top": "insufficient", "agreement": 1.0, "input_tokens": 215}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.78238205239177, "probs": [0.08689479609139293, 0.6606481239315956, 0.2524570799770115], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11071484163404, "probs": [0.8171805101277473, 0.07074519344147624, 0.11207429643077639], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93532188795507, "probs": [0.103178626971303, 0.7367489124270009, 0.16007246060169616], "top": "prohibited", "agreement": 1.0, "input_tokens": 209}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.05964413844049, "probs": [0.08560085550345033, 0.3403340342335013, 0.5740651102630484], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.45464807748795, "probs": [0.017814439341391167, 0.9671155236327101, 0.015070037025898734], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.7540820222348, "probs": [0.9513649777837366, 0.01716163808274369, 0.03147338413351958], "top": "B", "agreement": 1.0, "input_tokens": 201}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3223269302398, "probs": [0.025903351803450637, 0.9255875837884197, 0.048509064408129646], "top": "B", "agreement": 1.0, "input_tokens": 192}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.41108802519739, "probs": [0.019189428631759722, 0.9189139403005832, 0.061896631067657096], "top": "A", "agreement": 1.0, "input_tokens": 192}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.33608605712652, "probs": [0.016185500037668298, 0.9553471934068705, 0.028467306555461265], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.05230391956866, "probs": [0.014798671264711157, 0.9697705732073082, 0.015430755527980513], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.06105502322316, "probs": [0.025560818133635414, 0.022080003391839246, 0.9523591784745253], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.4657480455935, "probs": [0.05311118840492001, 0.6811115463281155, 0.2657772652669644], "top": "A", "agreement": 1.0, "input_tokens": 190}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.6061700694263, "probs": [0.026144179410345973, 0.9539350561208116, 0.019920764468842493], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27546603791416, "probs": [0.9592258084083156, 0.024179546083237823, 0.016594645508446512], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56313016079366, "probs": [0.015353470774838725, 0.9649135180343748, 0.019733011190786628], "top": "B", "agreement": 1.0, "input_tokens": 197}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.47261794656515, "probs": [0.6229250573303873, 0.29341037282098503, 0.08366456984862763], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.3645320981741, "probs": [0.9523442934756741, 0.028977529107373938, 0.018678177416952112], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.15088504366577, "probs": [0.9509486773226812, 0.026062332698517814, 0.022988989978801007], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.96097300387919, "probs": [0.028605829867980576, 0.9599959678670565, 0.01139820226496276], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.8348721601069, "probs": [0.027524206942156276, 0.10494883289705875, 0.8675269601607849], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2010251916945, "probs": [0.025763001405599038, 0.9598922160922182, 0.014344782502182778], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.45449790172279, "probs": [0.01775221293583573, 0.018510449440082843, 0.9637373376240814], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.04937507398427, "probs": [0.05805098351023624, 0.04331699912016453, 0.8986320173695993], "top": "B", "agreement": 1.0, "input_tokens": 200}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57214908860624, "probs": [0.40713917224854596, 0.04723599385140831, 0.5456248339000457], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.12470493651927, "probs": [0.9628824551152317, 0.020107611199832023, 0.017009933684936158], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.12464393489063, "probs": [0.017791047056264133, 0.965845589126288, 0.016363363817447876], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.00073395855725, "probs": [0.9697536223339248, 0.01200584747578968, 0.018240530190285528], "top": "A", "agreement": 1.0, "input_tokens": 194}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.58090887591243, "probs": [0.46166613268095874, 0.4246186674610029, 0.11371519985803841], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 192}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.32358607836068, "probs": [0.9434733126902809, 0.03465269924248234, 0.021873988067236654], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.62780003622174, "probs": [0.05653147014528852, 0.049865123655206005, 0.8936034061995054], "top": "supported", "agreement": 1.0, "input_tokens": 214}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.32153693586588, "probs": [0.9646005420197569, 0.019726610085808626, 0.01567284789443457], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.07094294391572, "probs": [0.011200303179937986, 0.9433281975088483, 0.045471499311213706], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2088250182569, "probs": [0.04435070832583449, 0.8823880770841831, 0.07326121458998254], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.23123496770859, "probs": [0.38436219601806726, 0.39248484865749256, 0.22315295532444018], "top": "prohibited", "agreement": 0.3333333333333333, "input_tokens": 229}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3120870385319, "probs": [0.038671196720481874, 0.051824945670009545, 0.9095038576095087], "top": "permitted", "agreement": 1.0, "input_tokens": 222}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55138920992613, "probs": [0.9386225473366789, 0.024670771221418437, 0.03670668144190261], "top": "insufficient", "agreement": 1.0, "input_tokens": 221}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.12138384021819, "probs": [0.06338502047709155, 0.20445210704639485, 0.7321628724765136], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.4868881367147, "probs": [0.01023906219590939, 0.9776572412337166, 0.012103696570374124], "top": "A", "agreement": 1.0, "input_tokens": 210}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.04196297936141, "probs": [0.08343895995982653, 0.048442981731284813, 0.8681180583088886], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.51782806962729, "probs": [0.028590681320234453, 0.07026846391461965, 0.9011408547651459], "top": "insufficient", "agreement": 1.0, "input_tokens": 211}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.9232379347086, "probs": [0.9485370017696566, 0.021991387483579136, 0.02947161074676416], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.87997285276651, "probs": [0.1476984273576982, 0.06533845399896182, 0.78696311864334], "top": "supported", "agreement": 1.0, "input_tokens": 227}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.09370393306017, "probs": [0.9659641119707486, 0.01300233393992522, 0.021033554089326202], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.80565191432834, "probs": [0.9452042665175415, 0.037745228086007734, 0.01705050539645064], "top": "insufficient", "agreement": 1.0, "input_tokens": 202}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.36551701463759, "probs": [0.7928775526060625, 0.1207272484983066, 0.08639519889563085], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63070995360613, "probs": [0.4902958520500278, 0.23093932363732236, 0.2787648243126498], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 231}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.61747888661921, "probs": [0.08132590401657058, 0.05465960600149562, 0.8640144899819338], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.39068693853915, "probs": [0.2290171921944468, 0.6937872957432885, 0.07719551206226469], "top": "insufficient", "agreement": 1.0, "input_tokens": 223}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.68264096602798, "probs": [0.030141465776599607, 0.9500195362761745, 0.019838997947225906], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.33379593491554, "probs": [0.025305433525381633, 0.03185065213808263, 0.9428439143365358], "top": "B", "agreement": 1.0, "input_tokens": 222}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63502989336848, "probs": [0.05059101952456338, 0.04279722164027021, 0.9066117588351665], "top": "B", "agreement": 1.0, "input_tokens": 215}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.9219030905515, "probs": [0.012768350746295443, 0.01860437029559649, 0.9686272789581081], "top": "insufficient", "agreement": 1.0, "input_tokens": 201}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.46073799580336, "probs": [0.9208322432392101, 0.04082517423414474, 0.038342582526645214], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.91627394407988, "probs": [0.8434852492825246, 0.05011598360172626, 0.10639876711574907], "top": "supported", "agreement": 1.0, "input_tokens": 219}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63296980783343, "probs": [0.023619578875477214, 0.01956735197412583, 0.9568130691503969], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56728013418615, "probs": [0.8650697558641954, 0.10248576421341996, 0.03244447992238454], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.89363302662969, "probs": [0.10549476831325036, 0.0754944842886127, 0.819010747398137], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.0271641947329, "probs": [0.335350091795104, 0.11069809870287566, 0.5539518095020204], "top": "prohibited", "agreement": 1.0, "input_tokens": 221}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.68310103937984, "probs": [0.8825022468959398, 0.07026933060840494, 0.047228422495655314], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49247793480754, "probs": [0.06932232571305925, 0.9077945737563922, 0.022883100530548473], "top": "insufficient", "agreement": 1.0, "input_tokens": 220}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.37710592709482, "probs": [0.009281733826954271, 0.9839356748468795, 0.006782591326166279], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.4181278925389, "probs": [0.008340312669325892, 0.9815921692199244, 0.010067518110749797], "top": "B", "agreement": 1.0, "input_tokens": 230}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.68250103481114, "probs": [0.012023447059675983, 0.9711751830662149, 0.016801369874109195], "top": "B", "agreement": 1.0, "input_tokens": 223}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.17658894322813, "probs": [0.011643930956106696, 0.9806920781093156, 0.007663990934577689], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.75012196972966, "probs": [0.900478485395825, 0.023178368052708644, 0.07634314655146633], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.1670160163194, "probs": [0.06499692312355161, 0.047496249800495424, 0.887506827075953], "top": "supported", "agreement": 1.0, "input_tokens": 224}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.39842785894871, "probs": [0.3922836361905562, 0.48352898160684454, 0.12418738220259931], "top": "insufficient", "agreement": 0.3333333333333333, "input_tokens": 220}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.264126021415, "probs": [0.0560499070635644, 0.020116182060737196, 0.9238339108756985], "top": "insufficient", "agreement": 1.0, "input_tokens": 208}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.99062398634851, "probs": [0.17500642472273137, 0.21124871005668264, 0.613744865220586], "top": "permitted", "agreement": 1.0, "input_tokens": 227}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.58249910920858, "probs": [0.5914884424684399, 0.035143500631624254, 0.37336805689993585], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 228}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.88512306660414, "probs": [0.9046569284481161, 0.042704941598692904, 0.05263812995319102], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.85170302167535, "probs": [0.029736313438628644, 0.033014009977779125, 0.9372496765835922], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.53259907849133, "probs": [0.7432028429041843, 0.03582480375679601, 0.2209723533390198], "top": "A", "agreement": 1.0, "input_tokens": 221}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55849915929139, "probs": [0.06632110060786917, 0.8329169964218126, 0.10076190297031828], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.30782693624496, "probs": [0.7397871533568318, 0.1860713523015022, 0.07414149434166593], "top": "A", "agreement": 1.0, "input_tokens": 207}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.69303103350103, "probs": [0.9248773731002589, 0.014113602375072627, 0.06100902452466855], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.64469003677368, "probs": [0.9556486965100951, 0.021248690458660385, 0.02310261303124464], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.4887479878962, "probs": [0.010853780626859795, 0.015814708468139885, 0.9733315109050004], "top": "supported", "agreement": 1.0, "input_tokens": 212}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.7502808291465, "probs": [0.9606317563420671, 0.018450800471885003, 0.020917443186047876], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.08011500537395, "probs": [0.8027819128710822, 0.011505476315571077, 0.18571261081334672], "top": "contradicted", "agreement": 0.6666666666666666, "input_tokens": 203}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55208909884095, "probs": [0.8487397319059693, 0.05482797840153974, 0.09643228969249089], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57966905273497, "probs": [0.019700876806489794, 0.943405397557902, 0.036893725635608235], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.4615489449352, "probs": [0.9481975719388965, 0.021083015776241384, 0.030719412284862045], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.54677893035114, "probs": [0.11594495476456004, 0.2790651305948641, 0.6049899146405758], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19642515666783, "probs": [0.011620895003629282, 0.009627189199890869, 0.9787519157964798], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.25867592170835, "probs": [0.9712341256474011, 0.015134214927072205, 0.013631659425526738], "top": "B", "agreement": 1.0, "input_tokens": 212}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.43758811056614, "probs": [0.9748621568921099, 0.014266993806726613, 0.010870849301163494], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.31377801857889, "probs": [0.01218702517662872, 0.9640155969735945, 0.02379737784977667], "top": "insufficient", "agreement": 1.0, "input_tokens": 191}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.83341207914054, "probs": [0.05216159441960384, 0.25033305566222464, 0.6975053499181716], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.97028390131891, "probs": [0.0744428867114499, 0.8598927356841719, 0.06566437760437807], "top": "supported", "agreement": 1.0, "input_tokens": 221}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.72164103016257, "probs": [0.031673836031138235, 0.010902017412968387, 0.9574241465558934], "top": "supported", "agreement": 1.0, "input_tokens": 229}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.75393207930028, "probs": [0.9598638711537073, 0.029206330829344665, 0.01092979801694802], "top": "insufficient", "agreement": 1.0, "input_tokens": 226}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.62590991705656, "probs": [0.3126088342455135, 0.527299092068065, 0.1600920736864215], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57155886292458, "probs": [0.018596715235427216, 0.8540525288330012, 0.1273507559315715], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.5972400829196, "probs": [0.2682095319185024, 0.5694223884119577, 0.16236807966953995], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.23564680665731, "probs": [0.5197715159798874, 0.040530340274099394, 0.4396981437460133], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 224}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.32945799827576, "probs": [0.03677949530401433, 0.9404844838908847, 0.02273602080510092], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49516899138689, "probs": [0.09722552658518739, 0.7089119171844285, 0.19386255623038412], "top": "B", "agreement": 1.0, "input_tokens": 218}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27444600686431, "probs": [0.0446355264117619, 0.029378934066784906, 0.9259855395214532], "top": "B", "agreement": 1.0, "input_tokens": 209}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.40609706752002, "probs": [0.8956127457317613, 0.023540301062398936, 0.08084695320583983], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.97919386439025, "probs": [0.9215420659013949, 0.05709241087032677, 0.021365523228278453], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.97835288010538, "probs": [0.024650674719169527, 0.05690076079826144, 0.9184485644825692], "top": "contradicted", "agreement": 1.0, "input_tokens": 215}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.66284011676908, "probs": [0.017881542110937673, 0.0314503320010902, 0.9506681258879721], "top": "contradicted", "agreement": 1.0, "input_tokens": 236}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55495896935463, "probs": [0.015971281239954956, 0.021403804942628207, 0.9626249138174169], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.41438700258732, "probs": [0.9469841068616602, 0.027062216114961006, 0.025953677023378714], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19617602787912, "probs": [0.9655262695437641, 0.017417092730978554, 0.01705663772525738], "top": "supported", "agreement": 1.0, "input_tokens": 241}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.8542031571269, "probs": [0.9523229876734454, 0.019887100792948465, 0.02778991153360623], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3565369695425, "probs": [0.02944395390697739, 0.022909170093210977, 0.9476468759998116], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76825093105435, "probs": [0.028467305657075486, 0.01618550022217239, 0.9553471941207522], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27273586578667, "probs": [0.9626180927581529, 0.027508993151357353, 0.009872914090489732], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.62328894250095, "probs": [0.01216177373585067, 0.025820060282101747, 0.9620181659820476], "top": "supported", "agreement": 1.0, "input_tokens": 216}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08963412232697, "probs": [0.008947931903485177, 0.02245643770922697, 0.9685956303872878], "top": "supported", "agreement": 1.0, "input_tokens": 237}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.0518649648875, "probs": [0.03490400868942244, 0.07256938070358598, 0.8925266106069916], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.60704015754163, "probs": [0.8560321676779682, 0.10355825471147521, 0.04040957761055657], "top": "contradicted", "agreement": 1.0, "input_tokens": 209}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.71373107470572, "probs": [0.9275028632254695, 0.04760344873379892, 0.024893688040731596], "top": "contradicted", "agreement": 1.0, "input_tokens": 230}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49781906977296, "probs": [0.9261949727483907, 0.021028996331540862, 0.0527760309200684], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.61885002627969, "probs": [0.03674381458625419, 0.02368412668428037, 0.9395720587294654], "top": "supported", "agreement": 1.0, "input_tokens": 222}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2677048612386, "probs": [0.026670715956627202, 0.020321964093249996, 0.9530073199501229], "top": "supported", "agreement": 1.0, "input_tokens": 243}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 115.50732096657157, "probs": [0.08330838841015804, 0.7031959025468014, 0.21349570904304058], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.2152119167149, "probs": [0.18172827933132565, 0.6645395854888777, 0.15373213517979664], "top": "prohibited", "agreement": 1.0, "input_tokens": 223}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.68451481312513, "probs": [0.08895150060315864, 0.816337598890086, 0.09471090050675535], "top": "prohibited", "agreement": 1.0, "input_tokens": 244}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.99779603257775, "probs": [0.0293692258945804, 0.11923451516955454, 0.851396258935865], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.88266407325864, "probs": [0.7559638022373625, 0.22951644325448917, 0.014519754508148302], "top": "permitted", "agreement": 1.0, "input_tokens": 220}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.7805620841682, "probs": [0.9180643670041458, 0.055699863652112935, 0.026235769343741142], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.50894907303154, "probs": [0.8545877320672847, 0.07877388866940015, 0.06663837926331509], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.93957284092903, "probs": [0.10813821274832802, 0.06970313933726799, 0.822158647914404], "top": "prohibited", "agreement": 1.0, "input_tokens": 219}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.30949609912932, "probs": [0.04460783183966404, 0.029981179056752127, 0.9254109891035839], "top": "prohibited", "agreement": 1.0, "input_tokens": 240}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.72408109530807, "probs": [0.9383472668786358, 0.03989758917731366, 0.021755143944050585], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27044597640634, "probs": [0.019454336721104887, 0.06822606373845527, 0.9123195995404397], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.04430316016078, "probs": [0.015551699476841353, 0.0471124973010445, 0.9373358032221141], "top": "permitted", "agreement": 1.0, "input_tokens": 246}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.75769089721143, "probs": [0.0377743579001357, 0.05503984914430352, 0.9071857929555608], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.15861508995295, "probs": [0.9057645436591656, 0.057300812015498664, 0.03693464432533565], "top": "prohibited", "agreement": 1.0, "input_tokens": 227}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.92353204265237, "probs": [0.9388430475426657, 0.037491202062820105, 0.023665750394514205], "top": "prohibited", "agreement": 1.0, "input_tokens": 248}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.23017605394125, "probs": [0.28824200219090185, 0.5868851532425197, 0.12487284456657842], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.60011996515095, "probs": [0.5631497490596038, 0.19793031332732086, 0.2389199376130753], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.13368405029178, "probs": [0.07442810557243966, 0.7426469601625133, 0.18292493426504697], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.7249421030283, "probs": [0.021424613092898774, 0.963560742046841, 0.01501464486026032], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.53416905552149, "probs": [0.01601169462902707, 0.9650607227828113, 0.01892758258816167], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.33336706086993, "probs": [0.01772358905716697, 0.9621833982502599, 0.020093012692573263], "top": "A", "agreement": 1.0, "input_tokens": 232}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.65838094428182, "probs": [0.038484166247206124, 0.9437641516667067, 0.017751682086087198], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.39987816102803, "probs": [0.01638885815779191, 0.9473306235405403, 0.036280518301667765], "top": "B", "agreement": 1.0, "input_tokens": 219}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63854097947478, "probs": [0.01830968230910091, 0.9532844759985375, 0.028405841692361555], "top": "B", "agreement": 1.0, "input_tokens": 240}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.11572505533695, "probs": [0.023557953254122476, 0.9543166602232142, 0.022125386522663444], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.6771310288459, "probs": [0.031413960782038314, 0.9495687268473656, 0.01901731237059622], "top": "A", "agreement": 1.0, "input_tokens": 214}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57792002893984, "probs": [0.07602225290100428, 0.8966935357050799, 0.027284211393915826], "top": "A", "agreement": 1.0, "input_tokens": 235}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.05223493278027, "probs": [0.023803230827026532, 0.0318997393139744, 0.944297029858999], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.40005697496235, "probs": [0.9537738809196197, 0.029021027054749942, 0.017205092025630454], "top": "B", "agreement": 1.0, "input_tokens": 214}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49492801167071, "probs": [0.964600540916227, 0.019726610712202018, 0.015672848371570857], "top": "B", "agreement": 1.0, "input_tokens": 235}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.36486602015793, "probs": [0.01331158113941839, 0.9684719956116256, 0.018216423248955952], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.95281415246427, "probs": [0.03555637420973125, 0.9480424533538477, 0.016401172436420974], "top": "A", "agreement": 1.0, "input_tokens": 219}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.867193011567, "probs": [0.02192392382604651, 0.9656109109209706, 0.012465165252982892], "top": "A", "agreement": 1.0, "input_tokens": 240}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.72509111464024, "probs": [0.019595497666027976, 0.022215172638678914, 0.9581893296952932], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76956083625555, "probs": [0.9596984415322811, 0.02225016024721796, 0.01805139822050088], "top": "B", "agreement": 1.0, "input_tokens": 217}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49475804530084, "probs": [0.9679216672147071, 0.013872260676720507, 0.01820607210857243], "top": "B", "agreement": 1.0, "input_tokens": 238}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56357905827463, "probs": [0.02350526733620643, 0.04401818593722302, 0.9324765467265705], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.89415200613439, "probs": [0.9307059696632914, 0.042134929049560656, 0.027159101287147974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.20111506432295, "probs": [0.9518944397002193, 0.030200951503065683, 0.01790460879671502], "top": "contradicted", "agreement": 1.0, "input_tokens": 248}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.41817701980472, "probs": [0.0734946189438627, 0.8668797160257922, 0.05962566503034527], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.31172591820359, "probs": [0.06641491931728377, 0.8517220237244112, 0.08186305695830509], "top": "prohibited", "agreement": 1.0, "input_tokens": 239}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.23841609433293, "probs": [0.026075603104276424, 0.9317425408247122, 0.04218185607101137], "top": "prohibited", "agreement": 1.0, "input_tokens": 260}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.47788806818426, "probs": [0.8272206536224163, 0.11345127657827278, 0.05932806979931101], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.95196408778429, "probs": [0.10398356939461242, 0.36466878809364145, 0.5313476425117462], "top": "A", "agreement": 0.3333333333333333, "input_tokens": 227}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.82776314206421, "probs": [0.047442131153065406, 0.31816121638654793, 0.6343966524603867], "top": "A", "agreement": 1.0, "input_tokens": 248}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.69308109208941, "probs": [0.026110501601006305, 0.021183264898242012, 0.9527062335007518], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.61059990525246, "probs": [0.9405169120595193, 0.024208959715331872, 0.035274128225148874], "top": "contradicted", "agreement": 1.0, "input_tokens": 237}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.1437259670347, "probs": [0.9538802773453129, 0.023059861528293313, 0.023059861126393875], "top": "contradicted", "agreement": 1.0, "input_tokens": 258}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.39161709696054, "probs": [0.11358428062013931, 0.09215012355248668, 0.7942655958273741], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.73537105321884, "probs": [0.6104725669434667, 0.2700592902850353, 0.11946814277149793], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 244}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.54128103144467, "probs": [0.8448473542913075, 0.08645996139090424, 0.06869268431778815], "top": "prohibited", "agreement": 1.0, "input_tokens": 265}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.34719696827233, "probs": [0.0265024583113502, 0.9469950831682415, 0.02650245852040832], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55919997952878, "probs": [0.04213492809402675, 0.9307059697792428, 0.027159102126730544], "top": "A", "agreement": 1.0, "input_tokens": 233}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.74452083185315, "probs": [0.02140380544189133, 0.962624913264518, 0.01597128129359056], "top": "A", "agreement": 1.0, "input_tokens": 254}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.96222283877432, "probs": [0.04515777084298974, 0.03741040508103143, 0.9174318240759788], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.6322501283139, "probs": [0.9100932546372905, 0.041201749446211355, 0.048704995916498045], "top": "contradicted", "agreement": 1.0, "input_tokens": 234}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.41820884868503, "probs": [0.9366656772497187, 0.038194709842211357, 0.025139612908070024], "top": "contradicted", "agreement": 1.0, "input_tokens": 255}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63193999789655, "probs": [0.868308675840529, 0.07676005878826689, 0.0549312653712041], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.79696197621524, "probs": [0.15789450752928103, 0.11538080349900755, 0.7267246889717114], "top": "prohibited", "agreement": 1.0, "input_tokens": 235}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76924092695117, "probs": [0.08083885614456791, 0.06032105358305889, 0.8588400902723732], "top": "prohibited", "agreement": 1.0, "input_tokens": 256}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.05577512271702, "probs": [0.009236476474052145, 0.9791380440055677, 0.011625479520380164], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.8926528096199, "probs": [0.014594034131288612, 0.9765710625943955, 0.008834903274315823], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.22870705649257, "probs": [0.0070908823458139675, 0.986522232315347, 0.006386885338839195], "top": "A", "agreement": 1.0, "input_tokens": 259}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.44164704717696, "probs": [0.07423043782078662, 0.012813700792315847, 0.9129558613868974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.67613986879587, "probs": [0.9080827305344887, 0.025413463174314043, 0.06650380629119718], "top": "contradicted", "agreement": 1.0, "input_tokens": 238}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.21236607059836, "probs": [0.912724042631006, 0.024497031790175153, 0.06277892557881881], "top": "contradicted", "agreement": 1.0, "input_tokens": 259}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.43738810904324, "probs": [0.44029144656423663, 0.4049592579435585, 0.15474929549220479], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 227}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.7701110150665, "probs": [0.36520001887064335, 0.21202778871080402, 0.4227721924185527], "top": "permitted", "agreement": 0.3333333333333333, "input_tokens": 238}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.33269806951284, "probs": [0.0870101345530018, 0.5040542344698868, 0.4089356309771113], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 259}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.88700200989842, "probs": [0.4770731173522989, 0.04585376828987368, 0.4770731143578275], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 221}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.543208938092, "probs": [0.39895641057043185, 0.0317669442524896, 0.5692766451770785], "top": "B", "agreement": 1.0, "input_tokens": 232}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.42057796940207, "probs": [0.3779951490486449, 0.03557902847506642, 0.5864258224762886], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 253}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.66607995517552, "probs": [0.02269220266989126, 0.01879907857817761, 0.9585087187519311], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.78358213044703, "probs": [0.9501164329419498, 0.021572126471092922, 0.028311440586957334], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.34352685883641, "probs": [0.9561058300755273, 0.022635315404623546, 0.021258854519849178], "top": "contradicted", "agreement": 1.0, "input_tokens": 245}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.68751108460128, "probs": [0.09448034891197682, 0.0384419614966505, 0.8670776895913728], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49676900357008, "probs": [0.7989049560014004, 0.057297274549302536, 0.143797769449297], "top": "prohibited", "agreement": 1.0, "input_tokens": 229}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.73400084488094, "probs": [0.9033714574378554, 0.03317980421079573, 0.06344873835134875], "top": "prohibited", "agreement": 1.0, "input_tokens": 250}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.24132514931262, "probs": [0.9807799827157122, 0.008509520309325268, 0.010710496974962463], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.576609890908, "probs": [0.009842123709939055, 0.01026250243418525, 0.9798953738558757], "top": "A", "agreement": 1.0, "input_tokens": 217}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.82468209415674, "probs": [0.011627702527445261, 0.00904705287357599, 0.9793252445989787], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.71378183178604, "probs": [0.6618883595686608, 0.28081044656836285, 0.05730119386297648], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.15876503288746, "probs": [0.06488320495606428, 0.37587178359700457, 0.5592450114469311], "top": "contradicted", "agreement": 1.0, "input_tokens": 235}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.83523188345134, "probs": [0.06100397866886853, 0.1887116244179442, 0.7502843969131873], "top": "contradicted", "agreement": 1.0, "input_tokens": 256}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.46343789994717, "probs": [0.17125015092489698, 0.5080506282282206, 0.32069922084688257], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.65554110892117, "probs": [0.1514529519738628, 0.627869626193126, 0.22067742183301117], "top": "permitted", "agreement": 1.0, "input_tokens": 237}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.08003607578576, "probs": [0.17013095481276255, 0.7202069629772057, 0.10966208221003176], "top": "permitted", "agreement": 1.0, "input_tokens": 258}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.21708515100181, "probs": [0.023818955965634256, 0.9449208465220887, 0.031260197512277065], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.51958803646266, "probs": [0.04244531776287583, 0.9375620734741275, 0.01999260876299673], "top": "A", "agreement": 1.0, "input_tokens": 224}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.52464907616377, "probs": [0.028930382730149296, 0.9507948329241313, 0.02027478434571944], "top": "A", "agreement": 1.0, "input_tokens": 245}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.82346696592867, "probs": [0.227340819830238, 0.7726591801697621], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.30589003302157, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.4612430203706, "probs": [0.16800751383797227, 0.8319924861620277], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.24332115240395, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.02690011076629, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.10359908640385, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.52028305083513, "probs": [0.20097170058635486, 0.7990282994136452], "top": "ordinary", "agreement": 1.0, "input_tokens": 254}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.98831695131958, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.21057002991438, "probs": [0.13950622912843208, 0.8604937708715679], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.29365990497172, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.78499589301646, "probs": [0.3067002008775075, 0.6932997991224925], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85053499415517, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.34242092818022, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "agreement": 1.0, "input_tokens": 256}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.2711100243032, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.47827295213938, "probs": [0.07740461434958126, 0.9225953856504188], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.6651051081717, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.16512987017632, "probs": [0.09196032674706488, 0.9080396732529351], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.14798917993903, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.846146998927, "probs": [0.1552608265624769, 0.844739173437523], "top": "ordinary", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.53142103925347, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.14905810542405, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.94955693557858, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8681258149445, "probs": [0.32019975839988934, 0.6798002416001107], "top": "ordinary", "agreement": 1.0, "input_tokens": 255}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7439940199256, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.34335108660161, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76852412521839, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.3001710139215, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.71955402009189, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.29695096611977, "probs": [0.13578298208575287, 0.8642170179142472], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.08391698077321, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.49642279930413, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.6236128937453, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98197806812823, "probs": [0.9710643563300382, 0.02893564366996184], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.12592894211411, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.64574500732124, "probs": [0.9673224283590844, 0.032677571640915566], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77144405432045, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.57463387213647, "probs": [0.9350120677910418, 0.06498793220895807], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.20593886449933, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28121114894748, "probs": [0.8405803537312423, 0.1594196462687578], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.936926105991, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.38287082873285, "probs": [0.9290542127600319, 0.07094578723996812], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.94771594367921, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36309093423188, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9001058023423, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36234098672867, "probs": [0.9516645319845058, 0.048335468015494064], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80190412327647, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.5760639179498, "probs": [0.9439168350492196, 0.056083164950780265], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.6807038988918, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.42426206357777, "probs": [0.905386513027929, 0.09461348697207088], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.92843992076814, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.01422993279994, "probs": [0.9156019647894573, 0.0843980352105427], "top": "react", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.05272791907191, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.68727384507656, "probs": [0.6155100736384166, 0.38448992636158347], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.71669416129589, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.72329600527883, "probs": [0.6932998086882517, 0.30670019131174836], "top": "react", "agreement": 1.0, "input_tokens": 268}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19860889017582, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43275200016797, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.15151889249682, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.50260305590928, "probs": [0.915601962671152, 0.08439803732884785], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97144604288042, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.35757098533213, "probs": [0.40698895046193323, 0.5930110495380667], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28612015023828, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.32979102991521, "probs": [0.36247441705939704, 0.637525582940603], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86123495921493, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.57758383639157, "probs": [0.908039679856927, 0.09196032014307312], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.13655905798078, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.41468108259141, "probs": [0.3481052260909569, 0.6518947739090432], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38678012415767, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.15712801553309, "probs": [0.5235093404303358, 0.4764906595696642], "top": "react", "agreement": 0.5, "input_tokens": 255}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.71493419446051, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.92836608737707, "probs": [0.2743951410804348, 0.7256048589195652], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89740589819849, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04450806602836, "probs": [0.2112354509623397, 0.7887645490376604], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16362794861197, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9370471779257, "probs": [0.12511350715617336, 0.8748864928438266], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06018781289458, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9700560439378, "probs": [0.27439514286499256, 0.7256048571350073], "top": "ordinary", "agreement": 1.0, "input_tokens": 181}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.93217612802982, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02686811797321, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.82853505946696, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.18568888492882, "probs": [0.4222145135481339, 0.5777854864518661], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.98884710669518, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.08308787085116, "probs": [0.1472092305510008, 0.8527907694489993], "top": "ordinary", "agreement": 1.0, "input_tokens": 183}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06857786513865, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90203596837819, "probs": [0.11201237833636817, 0.8879876216636319], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97265699505806, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.21939880400896, "probs": [0.12172004831180805, 0.878279951688192], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.08579801954329, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.26070994511247, "probs": [0.17243807104776526, 0.8275619289522348], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.22541002556682, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19753903336823, "probs": [0.16366828025224447, 0.8363317197477556], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.98509713634849, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.95649715140462, "probs": [0.3000708042310285, 0.6999291957689716], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76272408105433, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.32803106307983, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.15387104265392, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.78097607195377, "probs": [0.10592235643160026, 0.8940776435683997], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.50415300950408, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36565090715885, "probs": [0.19108583142179145, 0.8089141685782085], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.30175100266933, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06846796907485, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.10244797170162, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06581789068878, "probs": [0.7614507841997993, 0.23854921580020058], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.32965086586773, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.25039904005826, "probs": [0.7887645476620578, 0.2112354523379421], "top": "react", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.94458693265915, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02938687987626, "probs": [0.4921584130878123, 0.5078415869121877], "top": "ordinary", "agreement": 0.5, "input_tokens": 191}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06178805790842, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02292699366808, "probs": [0.293524032474578, 0.7064759675254221], "top": "ordinary", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0622178632766, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.93623599596322, "probs": [0.5156793119007109, 0.484320688099289], "top": "react", "agreement": 0.5, "input_tokens": 186}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.01688806526363, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.15817901305854, "probs": [0.6302458313412554, 0.3697541686587445], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86274486593902, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1518190111965, "probs": [0.7318060925310856, 0.2681939074689144], "top": "react", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.38249201327562, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97791687212884, "probs": [0.5469149637193962, 0.45308503628060387], "top": "react", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.81505595892668, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16654904186726, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9528968911618, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16597907245159, "probs": [0.44532390802961846, 0.5546760919703815], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.33500108122826, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.18584883958101, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.41027103736997, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.47817213647068, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "agreement": 1.0, "input_tokens": 195}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36457196995616, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.33312097378075, "probs": [0.2501313189948441, 0.749868681005156], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.91284606233239, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.47200305573642, "probs": [0.17696061199867663, 0.8230393880013234], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99424714781344, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.82125514373183, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.91911595873535, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.26298004388809, "probs": [0.2870615213045163, 0.7129384786954838], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.37156201153994, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.40243186242878, "probs": [0.4145813330827104, 0.5854186669172896], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19842914491892, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.41974212229252, "probs": [0.44532391331269805, 0.554676086687302], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43619184009731, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.38645199686289, "probs": [0.35525721497515955, 0.6447427850248404], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04506709240377, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.8875379320234, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.2403998952359, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.55362301319838, "probs": [0.8269315768853935, 0.12075476980815658, 0.05231365330644987], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3498358707875, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.50438414141536, "probs": [0.9608203282304785, 0.03917967176952151], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.78927601873875, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.51954312622547, "probs": [0.9620325019409397, 0.02528607877990563, 0.012681419279154706], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.54924081265926, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.03219002299011, "probs": [0.06890580795431694, 0.931094192045683], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63750396668911, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.88797603547573, "probs": [0.8320645593484999, 0.04643111322039759, 0.12150432743110248], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.82763112708926, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.38919304311275, "probs": [0.9544700034284429, 0.04552999657155723], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.13836791925132, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.63894403539598, "probs": [0.9527150999124248, 0.019483546419057494, 0.027801353668517746], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.25171701051295, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.56183596514165, "probs": [0.12172004535178665, 0.8782799546482134], "top": "holds", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.82846607267857, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.45110209286213, "probs": [0.0645643144235774, 0.035205281163747985, 0.9002304044126745], "top": "tavern", "agreement": 1.0, "input_tokens": 400}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.4450989253819, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.15949180722237, "probs": [0.1862839992992625, 0.8137160007007376], "top": "elsewhere", "agreement": 1.0, "input_tokens": 360}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.27982999756932, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.28468010388315, "probs": [0.36775256751831553, 0.024771247006193476, 0.6074761854754909], "top": "tavern", "agreement": 0.6666666666666666, "input_tokens": 412}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.37274796701968, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.7038659863174, "probs": [0.0796746792375593, 0.9203253207624406], "top": "holds", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.74966593831778, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.07725803554058, "probs": [0.08290919036094808, 0.8994500391535621, 0.017640770485489753], "top": "meet_wren", "agreement": 1.0, "input_tokens": 403}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.64616012945771, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.72556603513658, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 363}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.29604106396437, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.80744613334537, "probs": [0.6296525959845715, 0.35799812782967866, 0.012349276185749847], "top": "stay", "agreement": 0.6666666666666666, "input_tokens": 415}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.23984909802675, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.88931908644736, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 351}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.51656288281083, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.14508788846433, "probs": [0.5514055185500297, 0.41145265329172886, 0.03714182815824141], "top": "stay", "agreement": 1.0, "input_tokens": 404}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.97021405212581, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.74075683765113, "probs": [0.9673224283553485, 0.03267757164465143], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.50727294012904, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.72411488182843, "probs": [0.9603331749383929, 0.028023658411333054, 0.011643166650274083], "top": "stay", "agreement": 1.0, "input_tokens": 416}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.37858007289469, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.72998679056764, "probs": [0.7439389594906934, 0.2560610405093065], "top": "moved", "agreement": 1.0, "input_tokens": 345}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.02632701955736, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.8566652033478, "probs": [0.329927824572513, 0.5565122407492623, 0.11355993467822469], "top": "meet_wren", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.89645307138562, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.5058740247041, "probs": [0.02983020500731863, 0.9701697949926814], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.30081991292536, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.35860306583345, "probs": [0.7871268373024943, 0.18209101999191218, 0.030782142705593513], "top": "stay", "agreement": 1.0, "input_tokens": 410}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.19101706147194, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.60101514495909, "probs": [0.04979902878533396, 0.950200971214666], "top": "holds", "agreement": 1.0, "input_tokens": 334}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71721609868109, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.38942199572921, "probs": [0.9109478037576992, 0.057628718316645906, 0.03142347792565483], "top": "stay", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.8943829741329, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.90831899829209, "probs": [0.9719328658769364, 0.028067134123063546], "top": "same", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99407601729035, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.56396302208304, "probs": [0.9710720272034978, 0.01715460731299303, 0.01177336548350914], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.46507882140577, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.72345705516636, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.6033141836524, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.94037706404924, "probs": [0.1337836073050901, 0.8426345520770127, 0.0235818406178973], "top": "meet_wren", "agreement": 1.0, "input_tokens": 407}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.36020910553634, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.31413303129375, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.25228094123304, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.12661985680461, "probs": [0.7852826544019835, 0.20168841824349107, 0.013028927354525465], "top": "stay", "agreement": 1.0, "input_tokens": 419}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.01027598790824, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.58698500320315, "probs": [0.05776707671018535, 0.9422329232898147], "top": "holds", "agreement": 1.0, "input_tokens": 336}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.91218708269298, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.94969905726612, "probs": [0.706475960089318, 0.29352403991068193], "top": "stay", "agreement": 1.0, "input_tokens": 370}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71826500073075, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.47538392059505, "probs": [0.9631149772558724, 0.036885022744127596], "top": "same", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.90995703078806, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.00986016727984, "probs": [0.9583890947004433, 0.04161090529955677], "top": "stay", "agreement": 1.0, "input_tokens": 381}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.8569170460105, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.55049508623779, "probs": [0.05608316364845092, 0.9439168363515491], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.88351701572537, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.98493092134595, "probs": [0.5156793134278208, 0.4843206865721793], "top": "stay", "agreement": 0.5, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.5865438580513, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.57436488382518, "probs": [0.9471470205154783, 0.05285297948452184], "top": "same", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.09110795520246, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.63834714144468, "probs": [0.9471470206637833, 0.05285297933621657], "top": "stay", "agreement": 1.0, "input_tokens": 389}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.82459509931505, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.98955896496773, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 340}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.57765391841531, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.32298292405903, "probs": [0.7614507825686178, 0.23854921743138224], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.9044871404767, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.46674490347505, "probs": [0.9583890946533792, 0.04161090534662085], "top": "same", "agreement": 1.0, "input_tokens": 353}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.25261202640831, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.87496896646917, "probs": [0.9516645320340041, 0.04833546796599583], "top": "stay", "agreement": 1.0, "input_tokens": 385}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.92190799489617, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.51647387258708, "probs": [0.4921584129984156, 0.5078415870015844], "top": "holds", "agreement": 1.0, "input_tokens": 333}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.33158212713897, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.52993607148528, "probs": [0.07094578575062087, 0.9290542142493791], "top": "departure", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.25276103802025, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.9059280604124, "probs": [0.0820053924809899, 0.9179946075190101], "top": "elsewhere", "agreement": 1.0, "input_tokens": 346}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.24329088442028, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.46661498397589, "probs": [0.1724380702954841, 0.8275619297045159], "top": "departure", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.80831690877676, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.36799195036292, "probs": [0.26819391433993306, 0.7318060856600669], "top": "holds", "agreement": 1.0, "input_tokens": 339}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98376807011664, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.72154800780118, "probs": [0.04833546845435023, 0.9516645315456497], "top": "departure", "agreement": 1.0, "input_tokens": 373}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.894386947155, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.70648719370365, "probs": [0.06310762735784174, 0.9368923726421583], "top": "elsewhere", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.37505097873509, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.40918318368495, "probs": [0.17243806760921704, 0.8275619323907829], "top": "departure", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.91777578555048, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.90266687050462, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 337}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97770702280104, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.74062801338732, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "agreement": 1.0, "input_tokens": 371}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.06877919472754, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.5580259244889, "probs": [0.9608203281857868, 0.039179671814213245], "top": "same", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.40060290880501, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.53219615854323, "probs": [0.9439168372028508, 0.05608316279714923], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.28164111822844, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.44662391208112, "probs": [0.07740461992444837, 0.9225953800755516], "top": "holds", "agreement": 1.0, "input_tokens": 343}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.45911215431988, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.55080405250192, "probs": [0.6589789156058212, 0.3410210843941787], "top": "stay", "agreement": 1.0, "input_tokens": 377}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.20551907084882, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.51489504799247, "probs": [0.9596222367905244, 0.040377763209475684], "top": "same", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.50098302029073, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.95065103285015, "probs": [0.9422329206296081, 0.05776707937039187], "top": "stay", "agreement": 1.0, "input_tokens": 388}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.07782908901572, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.29138291627169, "probs": [0.061278160236647276, 0.9387218397633527], "top": "holds", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.64521508477628, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.28943295963109, "probs": [0.8910699865839312, 0.10893001341606875], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.30942200496793, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.25032299943268, "probs": [0.9663161960398492, 0.03368380396015082], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.75750488042831, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.94257886335254, "probs": [0.9673224283958624, 0.03267757160413767], "top": "stay", "agreement": 1.0, "input_tokens": 394}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.92820792645216, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.3852739688009, "probs": [0.8527907664515249, 0.14720923354847515], "top": "moved", "agreement": 1.0, "input_tokens": 348}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.43872310593724, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.01278894394636, "probs": [0.04161090472473705, 0.9583890952752631], "top": "outside", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.42055190727115, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.2709320038557, "probs": [0.05608316279714923, 0.9439168372028508], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.73327489383519, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.9559998512268, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "agreement": 1.0, "input_tokens": 390}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.66689417883754, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.36709385365248, "probs": [0.07094578723996812, 0.9290542127600319], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.32887197844684, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.43341496400535, "probs": [0.8040183777913431, 0.19598162220865686], "top": "stay", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.66176515258849, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.03382007032633, "probs": [0.959622237525547, 0.040377762474453], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.81577703543007, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.4074541144073, "probs": [0.9583890941641539, 0.041610905835846104], "top": "stay", "agreement": 1.0, "input_tokens": 396}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.64765398576856, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.87945801019669, "probs": [0.7781221683337017, 0.22187783166629826], "top": "moved", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.4133220501244, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.08631104044616, "probs": [0.21123545145303588, 0.7887645485469642], "top": "outside", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.01616808399558, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.3739150930196, "probs": [0.03688502246073314, 0.9631149775392668], "top": "elsewhere", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.85020686686039, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.88880888558924, "probs": [0.755706186939545, 0.244293813060455], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.78302614577115, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.39855393022299, "probs": [0.12172004614068278, 0.8782799538593171], "top": "holds", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.99120794050395, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.48658486828208, "probs": [0.8879876230204682, 0.11201237697953184], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.37772194296122, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.32739296928048, "probs": [0.9558139330893122, 0.04418606691068789], "top": "same", "agreement": 1.0, "input_tokens": 366}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71004491485655, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.75053891539574, "probs": [0.9631149782667298, 0.0368850217332701], "top": "stay", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.00204783678055, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.44089488126338, "probs": [0.09733498488397495, 0.9026650151160249], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.51378491148353, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.45474388077855, "probs": [0.8319924818822048, 0.16800751811779524], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.67342507839203, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.00283985771239, "probs": [0.9471470206637833, 0.05285297933621657], "top": "same", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0619480125606, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.29736207611859, "probs": [0.955813931654481, 0.044186068345518914], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.09145789965987, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}], "seconds": 50.6, "failures": 0}, "repeat": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04500105232, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 34.49}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44290489703417, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.65}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93786007352173, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.31}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3619741294533, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.67}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14155103079975, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.4}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39506402052939, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.67}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97822988033295, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.27}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67167800106108, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.89}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58376697078347, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.85}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.353283958509564, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.61}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.530665984377265, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.82}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.987529998645186, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 35.32}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55208686925471, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.82}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.774479016661644, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 35.04}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35782501846552, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 35.54}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.285215152427554, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 35.53}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72386792115867, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.97}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.06497110053897, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 35.21}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20986214466393, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.55}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.06577203795314, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 35.38}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.096070824190974, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.45}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16470208019018, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.45}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09525219164789, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.43}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42703516036272, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.77}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55763592198491, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.67}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63745701871812, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.67}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32582391239703, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.48}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37211390398443, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.46}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10782085545361, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.34}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56240592338145, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 34.78}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62248600833118, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.91}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.850690074265, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 35.02}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366505013778806, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.56}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.269542925059795, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.5}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39898488856852, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.67}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.947461009025574, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 35.13}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.983189871534705, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.25}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68570814281702, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.87}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95721901021898, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.29}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.88733994588256, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 35.19}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.153631914407015, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.4}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8617700189352, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.98}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71856799721718, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.88}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.542795995250344, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.77}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.073800107464194, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 34.35}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43395488709211, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.57}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.762477131560445, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.1}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47258591465652, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.69}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0768210850656, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.43}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.376095075160265, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.67}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15954185463488, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.39}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32599294744432, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.69}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96876997128129, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.26}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.304573925212026, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.53}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80917713791132, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.2}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.391364108771086, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.69}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.156810984015465, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.38}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11087116971612, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.49}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03248011879623, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.39}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.348624086007476, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.54}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06937120296061, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.49}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31664386577904, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.65}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06713113933802, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.43}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66463790088892, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.89}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94524895399809, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.27}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.097730906680226, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.48}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.167191971093416, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.51}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4609858263284, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.75}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65178797766566, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.85}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20965189859271, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.51}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36529382877052, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.68}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5681861769408, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.79}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.407753989100456, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.71}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.920970004051924, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 35.21}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39594481885433, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.76}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32396406121552, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.56}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02565003186464, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 34.35}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42365399375558, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.54}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95842903107405, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.21}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.024090928956866, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 35.28}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44269488379359, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.74}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87189000286162, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 35.06}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.110231816768646, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.37}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70768782123923, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 35.05}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45593503117561, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.75}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.885630037635565, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 35.02}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52934583462775, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.72}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.991070887073874, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 35.32}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26369305513799, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.46}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7491288818419, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.9}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32866398058832, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.51}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.938591092824936, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 35.1}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.451294949278235, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.5}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94837091118097, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 35.22}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5901460647583, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.79}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.95455093495548, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 35.28}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74252790026367, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.91}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.641567178070545, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.97}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41978504694998, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.74}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72334777750075, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 35.04}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51194593496621, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.77}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.17977290600538, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 35.39}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38049417734146, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.55}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.491695024073124, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.62}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04879991710186, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.25}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.422514986246824, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.51}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16603200882673, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.42}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48408518731594, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.69}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08914085291326, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.19}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66421787440777, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.79}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84656787663698, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.17}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40913490764797, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.72}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95990890264511, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.32}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28292300365865, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.57}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25927299819887, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.44}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40575397387147, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.7}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20939299091697, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.28}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.705407943576574, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.78}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1415710542351, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.48}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25442290306091, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.44}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09382098168135, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.31}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.083710078150034, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.42}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2714230325073, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.43}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45337505824864, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.56}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89451888203621, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.19}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.016761187464, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 35.14}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78203700110316, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.13}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32263413257897, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.58}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.001089891418815, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.21}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.393194157630205, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.7}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98326996527612, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.3}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.265892839059234, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.57}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89094912260771, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.16}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39977395161986, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.74}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82171809487045, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.14}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38731401972473, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.57}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30150288902223, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.56}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37119399011135, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.57}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15236089192331, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.42}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28553303517401, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.58}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95424902252853, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.19}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29558293707669, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.65}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75156692042947, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.06}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25324315018952, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.5}], "seconds": 5.4, "failures": 0}, "negative": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.99729009531438, "probs": [0.06566437538338366, 0.8598927385077062, 0.07444288610891006], "top": "insufficient", "option_logits": null, "prompt_sha256": "2111f6636d10660f740e0f032908924d0640e60a712060af19c6e8972d95e5c9", "input_tokens": 204, "server_ms": 34.34}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.38026390783489, "probs": [0.2945662267563794, 0.03956295740237862, 0.665870815841242], "top": "contradicted", "option_logits": null, "prompt_sha256": "ed5aa8d62ce8778e40b8c51f1d1fe90b1f392ab252247908adc4e84b2546ab08", "input_tokens": 202, "server_ms": 34.64}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.04976104199886, "probs": [0.9241406947470572, 0.051569175242271295, 0.024290130010671554], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6f696700120ee981b3e3e0dc237f67896db84867803273e124be96bfd72fecc", "input_tokens": 196, "server_ms": 34.23}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.27777302265167, "probs": [0.11215130215053613, 0.052825544241278395, 0.8350231536081855], "top": "contradicted", "option_logits": null, "prompt_sha256": "ef4b71a8ae0a1bc3711450f77606ddabd2be62c9c2b6587e1fcacbccfcb9cb00", "input_tokens": 196, "server_ms": 34.63}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.798418033868074, "probs": [0.15307015041158983, 0.014109646797217717, 0.8328202027911924], "top": "insufficient", "option_logits": null, "prompt_sha256": "215840810bc2483aa303cd79f5d4f92093a49aa9957dfa697f20989aa1390d55", "input_tokens": 209, "server_ms": 34.13}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.5531162135303, "probs": [0.03329916574926079, 0.8166100641505288, 0.1500907701002104], "top": "supported", "option_logits": null, "prompt_sha256": "fc0d6607f95e1b53fe2cfa889a2cf1bfed2a95e2da0424c85742050fd8b5cde7", "input_tokens": 206, "server_ms": 34.74}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.075750064104795, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "supported", "option_logits": null, "prompt_sha256": "f3bf4ddaa8f4178e0b5f59c09fffe0c07bfd4bd34d616bbf9577361257ca1afa", "input_tokens": 208, "server_ms": 34.2}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.37621405161917, "probs": [0.3541639151292647, 0.5160413123689802, 0.1297947725017551], "top": "contradicted", "option_logits": null, "prompt_sha256": "e5578ff7fa77b4e9f8593ede9b8e31ecccdca126f02444f3c94cdd0bb01ee6ca", "input_tokens": 200, "server_ms": 34.64}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.420075153931975, "probs": [0.017835936327419456, 0.8721495360638406, 0.11001452760873995], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b2ae5222808313b49d4b84a395fe3c5b48345949fa231e2c8fba100ed470a94", "input_tokens": 194, "server_ms": 34.46}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.42061508819461, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b42ed76cec9fa4a467694c0069d312883b296f2ee2385059235881dee283be", "input_tokens": 194, "server_ms": 34.63}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.329153856262565, "probs": [0.03186734083364, 0.0247947114952375, 0.9433379476711226], "top": "contradicted", "option_logits": null, "prompt_sha256": "d2c60f4c3e6615ad87fdb4cd474622ea0a7f3f3b3fd7df582b5539d02a9dbd1b", "input_tokens": 193, "server_ms": 34.59}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.80597005970776, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "contradicted", "option_logits": null, "prompt_sha256": "89f547d1f4dbfccb3c2e3443f18b4576344bd16efeaec8108d2febb0c913ec9e", "input_tokens": 192, "server_ms": 35.12}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.355073960497975, "probs": [0.03540454160362492, 0.924457767599498, 0.04013769079687701], "top": "contradicted", "option_logits": null, "prompt_sha256": "6beeb2f79fa6330dc68f2b769cf2779997d4b57c459bf40113bbc11a80353072", "input_tokens": 205, "server_ms": 34.64}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.778259022161365, "probs": [0.028539740573724572, 0.7452090392508762, 0.22625122017539928], "top": "insufficient", "option_logits": null, "prompt_sha256": "b094a32aab8f6f92e09b2405572effe6bec478d6d3b5985cdeb71dab794ed591", "input_tokens": 205, "server_ms": 34.99}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.14735200628638, "probs": [0.026796810283021767, 0.844600378585805, 0.12860281113117333], "top": "insufficient", "option_logits": null, "prompt_sha256": "4e199755434441474e22e920ad5f8587827e30212d91945271ceadbb26337008", "input_tokens": 199, "server_ms": 34.36}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.52001584507525, "probs": [0.1303439116251751, 0.013621023271142042, 0.8560350651036829], "top": "contradicted", "option_logits": null, "prompt_sha256": "d75aa0e009db26e5e38e3bfd553573eccf95a0526360da23bada050f23e908f8", "input_tokens": 194, "server_ms": 34.82}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.20521298237145, "probs": [0.1369403800499024, 0.018392355065480374, 0.8446672648846173], "top": "supported", "option_logits": null, "prompt_sha256": "9e994209df771b9f8829dd54c36907e41ed241a507cce230c7e3505302a9fa1e", "input_tokens": 198, "server_ms": 34.53}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.504266016185284, "probs": [0.7651520472632017, 0.10276696393112311, 0.13208098880567515], "top": "contradicted", "option_logits": null, "prompt_sha256": "45785396c292fe5cbe62d99a45714d49c993d6f5b5783769cec1795f7b84f35f", "input_tokens": 198, "server_ms": 34.8}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.98643995448947, "probs": [0.9249067812606918, 0.05495367443647756, 0.020139544302830666], "top": "contradicted", "option_logits": null, "prompt_sha256": "490eb806b0921b46aa816c18534ea40c0719e89eb76c34feb4e89d55c921bf36", "input_tokens": 194, "server_ms": 34.29}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.462075939401984, "probs": [0.0190993007707109, 0.7736996936098133, 0.2072010056194758], "top": "contradicted", "option_logits": null, "prompt_sha256": "7e29e327fc2aef321d690607a7c7347871c7eb3dd07ac6d45327aab094603d91", "input_tokens": 194, "server_ms": 34.75}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.80080804042518, "probs": [0.021206655711830842, 0.5200601388697108, 0.45873320541845836], "top": "insufficient", "option_logits": null, "prompt_sha256": "ffb27e8dfc1d8298a601b7f483c42104d68f6e8ebfea086896afd140756b8c6c", "input_tokens": 211, "server_ms": 34.05}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.34884411096573, "probs": [0.9097361205627902, 0.05076536678270011, 0.039498512654509606], "top": "supported", "option_logits": null, "prompt_sha256": "e7855c00f037441fcdc5ff3fefaa61ddaeb0e79e09ba6c0c6043c767ee6d98ca", "input_tokens": 211, "server_ms": 34.55}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.916579119861126, "probs": [0.9223434223370909, 0.04004589025644397, 0.03761068740646525], "top": "supported", "option_logits": null, "prompt_sha256": "1a0c7ecb563009263a7969e56c6e00fc34f99b8ad08b4db579ed9e2bc78f4c6f", "input_tokens": 201, "server_ms": 34.26}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.395883817225695, "probs": [0.1051237595843464, 0.007539451491827663, 0.8873367889238258], "top": "contradicted", "option_logits": null, "prompt_sha256": "610dae3d86ea66ae02b343c18c1478ef66be2619fe3a6c03ff05a434fac307d6", "input_tokens": 197, "server_ms": 34.59}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.20541181974113, "probs": [0.7409240241997488, 0.18635729477403776, 0.07271868102621346], "top": "insufficient", "option_logits": null, "prompt_sha256": "52c15cca48120ff357156a28827735982f0407b36b9806dc79a94193e52ed2a4", "input_tokens": 212, "server_ms": 34.28}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.338492926210165, "probs": [0.9070425995126343, 0.07852475778895956, 0.01443264269840612], "top": "prohibited", "option_logits": null, "prompt_sha256": "5df6c8de75514db23d6e53210ba8920e11a83a08e31ffd76867eb6d789db3f29", "input_tokens": 218, "server_ms": 34.68}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.18210197985172, "probs": [0.04968326826324694, 0.890344493113758, 0.05997223862299509], "top": "insufficient", "option_logits": null, "prompt_sha256": "3dc91e8f64358377c40956ef40870819732d6e66fc8ed44530ee8b7b1e32f1e6", "input_tokens": 203, "server_ms": 34.35}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.36336389556527, "probs": [0.09374299764974467, 0.009200493913293934, 0.8970565084369614], "top": "prohibited", "option_logits": null, "prompt_sha256": "866701419f4e54bef33f2151b0e4c0c830728dbfd94c6ba4a3fbd92b438151a2", "input_tokens": 209, "server_ms": 34.48}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.97902010753751, "probs": [0.20575303419639995, 0.025954061735184695, 0.7682929040684153], "top": "prohibited", "option_logits": null, "prompt_sha256": "6c14eaaad2ab386d458a6458fce2b79bcd81ecbfc3c66679c950d808dd0c2a3f", "input_tokens": 209, "server_ms": 34.26}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.40749415010214, "probs": [0.8315971903507323, 0.10489918128594154, 0.06350362836332618], "top": "permitted", "option_logits": null, "prompt_sha256": "d8b705bc82e40643b8dcd135b43f67a8e3671ecc2f28e81b48de6aa1cbb84792", "input_tokens": 214, "server_ms": 34.7}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.9850501883775, "probs": [0.258643657502053, 0.6628301324407543, 0.07852621005719278], "top": "insufficient", "option_logits": null, "prompt_sha256": "5000e3b05641e28823047872c8bd19f4604d76e0fa38c23bdad3d8e7be7bce61", "input_tokens": 209, "server_ms": 34.15}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.13424107991159, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "prohibited", "option_logits": null, "prompt_sha256": "97ddaad2b5074dab6f31a5a7a8ded12c3e51123aa5d819f19c006ba123fde089", "input_tokens": 209, "server_ms": 34.44}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.14440111070871, "probs": [0.09658208517581993, 0.868022296339126, 0.03539561848505407], "top": "insufficient", "option_logits": null, "prompt_sha256": "5783bf781c97c761780f8d8454e12c8895251116d4d5ebe264401d9d9dcae0a1", "input_tokens": 208, "server_ms": 34.36}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.70290804095566, "probs": [0.0651099549445377, 0.02705164492548852, 0.9078384001299739], "top": "prohibited", "option_logits": null, "prompt_sha256": "a92081d320caa09114cab155b336249985672a93e86328503ed4397aa61c3293", "input_tokens": 211, "server_ms": 34.59}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.96510987356305, "probs": [0.049132112383896635, 0.729412594713007, 0.22145529290309635], "top": "insufficient", "option_logits": null, "prompt_sha256": "2b0f13909e189564a5e5099e9094600d4804ae6ef7ef59e7435f85302977a0c0", "input_tokens": 202, "server_ms": 34.3}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.599377101287246, "probs": [0.1638594512396548, 0.04974899459992729, 0.7863915541604178], "top": "prohibited", "option_logits": null, "prompt_sha256": "a4ffa08e79c520845fb80c116412fe4c48304f5f003b514bbcbea0a7c61cfe7b", "input_tokens": 204, "server_ms": 34.9}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.21099207177758, "probs": [0.06162034871506134, 0.8591822512326693, 0.07919740005226927], "top": "insufficient", "option_logits": null, "prompt_sha256": "f51b6f4e075e67f76ff574d496ba2f9c19cc2ca38a2f56d989791df67251d344", "input_tokens": 214, "server_ms": 34.37}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.37342404201627, "probs": [0.7621344563262837, 0.23138990123980285, 0.006475642433913465], "top": "insufficient", "option_logits": null, "prompt_sha256": "baf1674ea5ff54f508700c815711c06e9a50808663840042c147d883c72021ed", "input_tokens": 214, "server_ms": 34.66}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.8067380040884, "probs": [0.14398191537109406, 0.021923360098142518, 0.8340947245307634], "top": "insufficient", "option_logits": null, "prompt_sha256": "8854be3865426f818cb7189838d63dc61b9b64da67d13087c8e6ccc2a708a8c0", "input_tokens": 213, "server_ms": 34.11}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.663517985492945, "probs": [0.2129289153471605, 0.545676239155193, 0.24139484549764645], "top": "permitted", "option_logits": null, "prompt_sha256": "913468ed99926586c9292a6dabcee9b083a7b1b1c719b6bad31fda7f7f045a2e", "input_tokens": 212, "server_ms": 34.98}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.585297133773565, "probs": [0.15943768499721103, 0.025848589203191098, 0.814713725799598], "top": "insufficient", "option_logits": null, "prompt_sha256": "acfb724b1ffcab3d0820c0a1f4cdb189396280c1faf4e622ce1f306cad1bf45e", "input_tokens": 216, "server_ms": 34.9}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.80915914103389, "probs": [0.9471510047644319, 0.04112297491157286, 0.011726020323995183], "top": "insufficient", "option_logits": null, "prompt_sha256": "7a9df7814cddba377022c022111d998db3d8a2ff48c109eb7f1aac0f5c151319", "input_tokens": 216, "server_ms": 35.11}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.04388998448849, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "ecd27e22648cba485966b925a1b12fd5a888331b66634b0f8919798224cf8072", "input_tokens": 208, "server_ms": 34.34}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 38.18748309276998, "probs": [0.2729138208779564, 0.3294318709588002, 0.3976543081632435], "top": "insufficient", "option_logits": null, "prompt_sha256": "86759c52a7cc961194391587ca2df8d3998c59ec236637e4acc083f151ad5242", "input_tokens": 215, "server_ms": 35.31}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.773448111489415, "probs": [0.5472741025611921, 0.331307576591957, 0.12141832084685092], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ee81091d38ca686274af8a9057ae962583e07c84d4a7c8913246f20a9adc153", "input_tokens": 209, "server_ms": 34.1}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.6733981538564, "probs": [0.10569064817366915, 0.056437743537509265, 0.8378716082888217], "top": "insufficient", "option_logits": null, "prompt_sha256": "d890e328d92ed0294fd0ca0f45891d715f1b53fae8d48848ee13c936721cc9f2", "input_tokens": 212, "server_ms": 34.9}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.76402708515525, "probs": [0.745864682612535, 0.0779433517872937, 0.17619196560017134], "top": "permitted", "option_logits": null, "prompt_sha256": "866975030f9845962316982b48126dfbaa42cd89b8d3719305c5c1cd2a7ce35c", "input_tokens": 209, "server_ms": 34.11}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.45305398479104, "probs": [0.30671712087126574, 0.5394587819774863, 0.153824097151248], "top": "permitted", "option_logits": null, "prompt_sha256": "33ed49b156b02a8251a71c89906619f22d3cff122755a441bc701809f452ec03", "input_tokens": 210, "server_ms": 34.77}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.712218007072806, "probs": [0.9411304198850982, 0.03837677130689544, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "26a5967a0d4f86a5e648fdf739a14ae8463a48c7dc0319e4b9026afcfc70ce49", "input_tokens": 200, "server_ms": 34.97}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.77636890299618, "probs": [0.2191677303480843, 0.5616645393038313, 0.2191677303480843], "top": "insufficient", "option_logits": null, "prompt_sha256": "5cb564ef00f9a81eb96ac41fe01780b4b9ffdcb60d7bc8217b9d89860c279ba3", "input_tokens": 201, "server_ms": 34.97}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.37250319682062, "probs": [0.8246471790493917, 0.16138164146161363, 0.013971179488994609], "top": "insufficient", "option_logits": null, "prompt_sha256": "e747a01e40c9029fd1004867b0db224405c02f421c64f74cd748920300cb2000", "input_tokens": 192, "server_ms": 34.69}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.469625007361174, "probs": [0.9136677967042812, 0.07428832300095413, 0.012043880294764588], "top": "B", "option_logits": null, "prompt_sha256": "3d5649d07c298093a047c4d77967454e627c1fd4792f41873c4640c036a163cd", "input_tokens": 192, "server_ms": 34.67}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.36080415546894, "probs": [0.807231387784515, 0.17909249095841714, 0.013676121257067941], "top": "insufficient", "option_logits": null, "prompt_sha256": "88ac81fbf23c9684447ca373863f9cf4c54ca0bd35eb3e960680db4187456d96", "input_tokens": 208, "server_ms": 34.68}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.361623952165246, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "insufficient", "option_logits": null, "prompt_sha256": "7263692449eeef173103012af0a80bd27db0a89dc35bf50d4fed73b80a29f045", "input_tokens": 211, "server_ms": 34.62}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.532604997977614, "probs": [0.046119873005046885, 0.9369780144263187, 0.016902112568634386], "top": "B", "option_logits": null, "prompt_sha256": "4c2c0840c1ad7e181d30f10eb62a4618c3195e936034bc7274fe3c8fc945d786", "input_tokens": 200, "server_ms": 34.78}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.64344705268741, "probs": [0.6742384798146065, 0.29826788550402233, 0.02749363468137107], "top": "B", "option_logits": null, "prompt_sha256": "2d0f22653ae25b1b9ad253802809222e0554c7bd42292a5a1637a04128e5c52f", "input_tokens": 190, "server_ms": 34.83}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.852548994123936, "probs": [0.8763745016908776, 0.09751141040687612, 0.026114087902246324], "top": "insufficient", "option_logits": null, "prompt_sha256": "676f3a162c1d11c22fd5f07e5b57e6b6e6d3a4aae515a7c2965aeac124b616dd", "input_tokens": 203, "server_ms": 35.07}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.74216817691922, "probs": [0.13968031972950484, 0.23073250176366764, 0.6295871785068274], "top": "insufficient", "option_logits": null, "prompt_sha256": "109526bdb944267ab212d1b8ba2f77a3a6c7c147016cf89ae5bc2e0e8e661adc", "input_tokens": 206, "server_ms": 35.05}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.36689384095371, "probs": [0.9673249441344611, 0.025425185481267775, 0.007249870384270957], "top": "insufficient", "option_logits": null, "prompt_sha256": "c4889b691603599fd8dca223a645f5e88b7248a364a9bef64c37a8a9e3ced5eb", "input_tokens": 197, "server_ms": 34.66}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.86537004634738, "probs": [0.49248819366063284, 0.16951261805505752, 0.3379991882843097], "top": "insufficient", "option_logits": null, "prompt_sha256": "5deb35c130c31229df54964339f9faccffa08f3ed2eb38730d86ac9ccd510aa3", "input_tokens": 192, "server_ms": 35.19}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.495895056053996, "probs": [0.1558415388693792, 0.330859461294187, 0.5132989998364338], "top": "insufficient", "option_logits": null, "prompt_sha256": "7c27c7c3361bd04c584a975218ab6b610dcab0f9bf6078cf3870b470410c383c", "input_tokens": 203, "server_ms": 34.69}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.85022906959057, "probs": [0.30165258046080173, 0.13344430167683133, 0.5649031178623669], "top": "B", "option_logits": null, "prompt_sha256": "27dba9d4c38e8b61907893c325f5851e4726ba89fa7ac9e5563a14abe89239b9", "input_tokens": 203, "server_ms": 35.18}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.35932405106723, "probs": [0.9480355945752582, 0.038658346366163006, 0.013306059058578745], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f3dd4d0a864d976d5c35a7b088b3aed4b55500043acc8701dd4d35729125db8", "input_tokens": 193, "server_ms": 34.67}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.934010149911046, "probs": [0.22389674812152066, 0.7374540645868362, 0.03864918729164314], "top": "A", "option_logits": null, "prompt_sha256": "164c8915823587dee72ef8800598282728e580a02992fe7a8d8fa557ead4e3b9", "input_tokens": 192, "server_ms": 35.25}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.480314960703254, "probs": [0.9379420530362722, 0.04616732447007806, 0.01589062249364978], "top": "insufficient", "option_logits": null, "prompt_sha256": "d7c03673e95db552913947e72213d55f0938d187ca0d4fd69182cbe141bfda65", "input_tokens": 208, "server_ms": 34.76}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.83661010675132, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "A", "option_logits": null, "prompt_sha256": "b54ee64662a21acf112153e17512f10436c1938df35ab9ad182be1ff0c47c3e0", "input_tokens": 206, "server_ms": 34.99}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.46568597853184, "probs": [0.13579878113602123, 0.8376256951826758, 0.02657552368130303], "top": "A", "option_logits": null, "prompt_sha256": "8aa1b7f158bf57460e6f9f057913e0d01d913f16998cc0a46327e8aa6c93e838", "input_tokens": 200, "server_ms": 34.72}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.85776998847723, "probs": [0.25821814633948337, 0.25821814633948337, 0.4835637073210332], "top": "A", "option_logits": null, "prompt_sha256": "b17c06e9ca0e91accd4cfcb4667c5c3a42687d3209e1ecd98910eb1343a18dc2", "input_tokens": 193, "server_ms": 34.94}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.37217397429049, "probs": [0.12660014777476133, 0.09251256875552054, 0.7808872834697181], "top": "insufficient", "option_logits": null, "prompt_sha256": "324bb2046ede92322b8175d6af2961ad208eba5f52712692465ef944de2193a4", "input_tokens": 206, "server_ms": 34.66}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.625736789777875, "probs": [0.9299397393693557, 0.06263931303365462, 0.007420947596989686], "top": "insufficient", "option_logits": null, "prompt_sha256": "86e7d75bb9b41e2d9de272239185c5cce5c19dc1aac6573a737c31d47ab0563f", "input_tokens": 209, "server_ms": 34.9}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.43445500731468, "probs": [0.37290349783252347, 0.01111172675798736, 0.6159847754094891], "top": "insufficient", "option_logits": null, "prompt_sha256": "a1a07677dc816ad01403f9cff2867d5bfdaff2c29d8a4c2293ce0662a24f33ae", "input_tokens": 194, "server_ms": 34.75}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.81815990805626, "probs": [0.475785940079552, 0.1976777825029813, 0.3265362774174668], "top": "insufficient", "option_logits": null, "prompt_sha256": "82b8973ae0b97456180d256ef08822a11d20365d8039148ed863e9bb75c9bda4", "input_tokens": 192, "server_ms": 35.1}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.53373585641384, "probs": [0.14585095569321507, 0.009226876843769601, 0.8449221674630154], "top": "insufficient", "option_logits": null, "prompt_sha256": "9a672222518ab31da9c3ab2072849ef03b511e8e9d23762f89d33a83feca6ec1", "input_tokens": 216, "server_ms": 34.86}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.65196702443063, "probs": [0.09629593898139954, 0.7633942252849403, 0.14030983573366013], "top": "contradicted", "option_logits": null, "prompt_sha256": "5d2348206184a53d2da714668c8004522fc660c04abc09a4675170278b6d0ce8", "input_tokens": 214, "server_ms": 34.92}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.61266707442701, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "497e689287006b51bd4f178ebc9023050c338fca8abf790eef2112975de7f243", "input_tokens": 209, "server_ms": 34.9}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.73092804476619, "probs": [0.9369780144263187, 0.046119873005046885, 0.016902112568634386], "top": "supported", "option_logits": null, "prompt_sha256": "984a11157ce42cd37ec6209d96b53bf0d293025d6b69161f1a358f9bcc9d57db", "input_tokens": 207, "server_ms": 34.97}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.074760999530554, "probs": [0.6452505747343799, 0.2680863307491953, 0.08666309451642484], "top": "permitted", "option_logits": null, "prompt_sha256": "ea5a71f7bf051b93ca522c5ff3caef6ffdfcb5011f24af51a404b10524b250cb", "input_tokens": 228, "server_ms": 34.19}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.42403397336602, "probs": [0.43020843462499875, 0.1903144997976119, 0.3794770655773894], "top": "permitted", "option_logits": null, "prompt_sha256": "b5ac43b527f17cc117db3df096bff98320c5a248422c7b513eedeba99e619731", "input_tokens": 229, "server_ms": 34.76}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.098780972883105, "probs": [0.056437743537509265, 0.8378716082888217, 0.10569064817366915], "top": "insufficient", "option_logits": null, "prompt_sha256": "be77461c7c49b38137b436f74bcc533cf9fdcc1fbbcabe3acbe8c8c37f2272bd", "input_tokens": 222, "server_ms": 34.34}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.83157980069518, "probs": [0.1110623891201585, 0.008481107191351438, 0.88045650368849], "top": "permitted", "option_logits": null, "prompt_sha256": "12c28c53461194e05d3490b99aecb157c1965b942f5bfee5ef8163543d2bb1d0", "input_tokens": 221, "server_ms": 34.92}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.077460903674364, "probs": [0.7598853370160624, 0.23070704972212444, 0.009407613261813141], "top": "insufficient", "option_logits": null, "prompt_sha256": "0b7bc0317c30767061abcd7c39f9b71942a3c49b230cf334d1e6db7873b4dc61", "input_tokens": 216, "server_ms": 34.34}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.851809058338404, "probs": [0.9718671022270775, 0.019875215614779472, 0.008257682158143088], "top": "B", "option_logits": null, "prompt_sha256": "282f482d7bf4f65f4cc0f8dd6ca05d597f32f80783d49b010173ef79b130ffc5", "input_tokens": 210, "server_ms": 35.04}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.18440211378038, "probs": [0.08284327606718382, 0.0730741626697933, 0.8440825612630228], "top": "A", "option_logits": null, "prompt_sha256": "6bf8f97ede01d029938070b85d9e3aa545cbbbb84007fb406100f0a0a8ffa474", "input_tokens": 209, "server_ms": 34.53}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.27159299887717, "probs": [0.1753450388599366, 0.7903403209860906, 0.03431464015397285], "top": "A", "option_logits": null, "prompt_sha256": "33dc6c8523ece68e76db4eaffbb8a8470964368747977448381041503e3ad36c", "input_tokens": 211, "server_ms": 35.16}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.11166006885469, "probs": [0.03629282833642113, 0.01605512937357001, 0.9476520422900088], "top": "supported", "option_logits": null, "prompt_sha256": "6e4467088790119e033569f3066af7eedf2f72c934657e3a20082f685de045f2", "input_tokens": 226, "server_ms": 34.65}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.113551050424576, "probs": [0.044616056760944024, 0.6623675120954924, 0.2930164311435636], "top": "insufficient", "option_logits": null, "prompt_sha256": "1a718aab5e212abb4604d4e57ad3d7fb8cba756be66eba7d483eec463cb18c6b", "input_tokens": 227, "server_ms": 35.32}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.33665309846401, "probs": [0.04013769079687701, 0.03540454160362492, 0.924457767599498], "top": "contradicted", "option_logits": null, "prompt_sha256": "bafd2feb34ff693920168afa57cc5bc050eea256c82f2e9e071467805a7857f4", "input_tokens": 202, "server_ms": 34.79}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.83228900283575, "probs": [0.21057073282474073, 0.0031468025509083547, 0.7862824646243509], "top": "supported", "option_logits": null, "prompt_sha256": "c4c6394f10ee2df26d3dac75e02000ae56f28719ec24b10add7af6e77f5a4889", "input_tokens": 202, "server_ms": 35.14}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.52517607063055, "probs": [0.10502328955939726, 0.2865713245281423, 0.6084053859124604], "top": "permitted", "option_logits": null, "prompt_sha256": "37bd9a494e79ba1e91bbf2a68704609c66421b3ec11ab8a5bdd36908b17135b1", "input_tokens": 233, "server_ms": 34.76}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.81800903379917, "probs": [0.2680863307491953, 0.08666309451642484, 0.6452505747343799], "top": "insufficient", "option_logits": null, "prompt_sha256": "a115c9f1aaee5b00fa1fd3be52a599afa7112fdcf9eec8871ae98b3d6ef2183f", "input_tokens": 231, "server_ms": 35.13}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.50865487381816, "probs": [0.07184163899867897, 0.8298456430479688, 0.09831271795335216], "top": "prohibited", "option_logits": null, "prompt_sha256": "100e5f7a276a412d51e2ddf940777a1ff109ea7803a8291d638760c392064d80", "input_tokens": 216, "server_ms": 34.71}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 38.05769211612642, "probs": [0.7469915530638082, 0.14618461002062086, 0.106823836915571], "top": "prohibited", "option_logits": null, "prompt_sha256": "6486b8dc13e88f4d5e489f3b3b76c6bb5aeed63f19005132e10380f0181726aa", "input_tokens": 223, "server_ms": 35.36}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.593826185911894, "probs": [0.8638851317960057, 0.051328051461926705, 0.08478681674206763], "top": "B", "option_logits": null, "prompt_sha256": "941cd3d853797aa1a92c8261e9339af72809062c590b26ef2db8bdbbc5d68191", "input_tokens": 222, "server_ms": 34.78}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.742887856438756, "probs": [0.046253998446132885, 0.9397029314342114, 0.014043070119655722], "top": "A", "option_logits": null, "prompt_sha256": "f9c6ae0da8d46bec4df2b4102f3aebb45748add11af2d63a981d879fea6a717c", "input_tokens": 222, "server_ms": 35.02}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.383273942396045, "probs": [0.04067015270682889, 0.7760155022030047, 0.1833143450901663], "top": "insufficient", "option_logits": null, "prompt_sha256": "4df509501a3fa1e95f377f6b2cd7ff2737ebe52dd6e95472a1a35d0fc43e3fdf", "input_tokens": 215, "server_ms": 34.74}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.88223001174629, "probs": [0.05543534738188995, 0.9330136575005004, 0.01155099511760962], "top": "A", "option_logits": null, "prompt_sha256": "d68329bad204407dddf0266c2d55c897537804b87978786e2c29649197957a18", "input_tokens": 201, "server_ms": 35.15}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.58872696198523, "probs": [0.06878691906304032, 0.03042978117418096, 0.9007832997627787], "top": "supported", "option_logits": null, "prompt_sha256": "3fdb716c314c67c93413daa3cd5a38b13bec265f706d90a97c43e5a108104f13", "input_tokens": 223, "server_ms": 34.76}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.44301479309797, "probs": [0.06893683236028562, 0.08321304657583695, 0.8478501210638775], "top": "contradicted", "option_logits": null, "prompt_sha256": "d68195061969b189d71df192da0191d4d56877cf1d8ce0d4479bc8898c998c94", "input_tokens": 219, "server_ms": 34.68}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.92076890729368, "probs": [0.016115540131028547, 0.8390475816237557, 0.14483687824521582], "top": "insufficient", "option_logits": null, "prompt_sha256": "6edd6bca63afcb6fb6d263e3e248e24e556fde2b01d965155729342ef37e411e", "input_tokens": 220, "server_ms": 34.07}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.58401679806411, "probs": [0.2862501142494634, 0.02478133063598072, 0.6889685551145559], "top": "contradicted", "option_logits": null, "prompt_sha256": "d113705714c5c456d527c8466aeda96099d32a8069c13bf0f40b1ff3ed43ad11", "input_tokens": 218, "server_ms": 34.73}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.04240103252232, "probs": [0.1609098519153101, 0.7722359149508133, 0.06685423313387646], "top": "insufficient", "option_logits": null, "prompt_sha256": "19fd16742b6e54b9e2452e09592de10d5fc44c96cab1630e5f7651b1af77d898", "input_tokens": 224, "server_ms": 34.26}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 38.01392111927271, "probs": [0.3043910694191852, 0.39121786116162954, 0.3043910694191852], "top": "insufficient", "option_logits": null, "prompt_sha256": "ea512df6fc353426e37889a53cae2118b9e832157a4af067cb4a85489a5c83ff", "input_tokens": 221, "server_ms": 35.18}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.33347402885556, "probs": [0.08097077759111387, 0.04060828666999563, 0.8784209357388906], "top": "prohibited", "option_logits": null, "prompt_sha256": "71ad1455eaea177dccc5d9fc4078b19378e2ce1c1d27289d9120a1c445349487", "input_tokens": 221, "server_ms": 34.69}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.67030709423125, "probs": [0.8979988326296015, 0.06048782007179681, 0.041513347298601795], "top": "permitted", "option_logits": null, "prompt_sha256": "31939a3c71e268cd4751d57ccd6a07ff44f1ca1f81c7c2f7bf165ea50668fe0a", "input_tokens": 220, "server_ms": 34.94}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.71892795339227, "probs": [0.9821934763214742, 0.008345422472445314, 0.009461101206080565], "top": "B", "option_logits": null, "prompt_sha256": "13f4ee2ce11c253ccf784222a18f900b57830384e577f4adf0030d9893ed01a9", "input_tokens": 227, "server_ms": 34.85}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.50155516900122, "probs": [0.9584447268648851, 0.039082802533792566, 0.0024724706013222815], "top": "insufficient", "option_logits": null, "prompt_sha256": "c8a6bf0c918df98324f40d3c989bf2d3593c8114ebaba9510a0de7a63d90c43f", "input_tokens": 230, "server_ms": 34.66}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.197882076725364, "probs": [0.8152011530118691, 0.1808606650321593, 0.003938181955971634], "top": "insufficient", "option_logits": null, "prompt_sha256": "d512a04940165a8dc92bcf5e8a905fa84148aca8078cba93d34897ca73ebe5fc", "input_tokens": 223, "server_ms": 34.31}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.49279584735632, "probs": [0.9509567466006987, 0.028336479497072638, 0.020706773902228682], "top": "A", "option_logits": null, "prompt_sha256": "726c7f274cbd758a0855ced1b8e06e30b0888e289eeb459d087a868fb9610ef5", "input_tokens": 210, "server_ms": 34.7}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.237151991575956, "probs": [0.03787087428859695, 0.033405034339472804, 0.9287240913719302], "top": "supported", "option_logits": null, "prompt_sha256": "0032df826a7e49e7be6e1c8d5f96ece8822a858eeb6f5280c55a79fe2e628bbb", "input_tokens": 227, "server_ms": 34.28}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.58514695800841, "probs": [0.04458832792770586, 0.7990410839770169, 0.15637058809527726], "top": "insufficient", "option_logits": null, "prompt_sha256": "c29cf273a3d78467e10369969a81a42df358b37cb972ad44f78d455f8785b7e8", "input_tokens": 224, "server_ms": 34.66}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.142971996217966, "probs": [0.6079293570442296, 0.046423581791889615, 0.34564706116388083], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b9e9fd277a7252eb95c0dea4201d1d75394e8dd03ee09f576f6fc0355881f3", "input_tokens": 220, "server_ms": 34.26}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.706168135628104, "probs": [0.07421350543321283, 0.7101726109045401, 0.2156138836622472], "top": "contradicted", "option_logits": null, "prompt_sha256": "25fc9b80402930a472ddf5fbb9527f22c905747ffefc59e4865da0bcf422e5bb", "input_tokens": 208, "server_ms": 34.72}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.24985313601792, "probs": [0.3596689227288795, 0.5240624893078483, 0.11626858796327226], "top": "prohibited", "option_logits": null, "prompt_sha256": "1a9834724cc432424395c6e3e4989cae7698e00f85fedac3d9a8cd1a698cd265", "input_tokens": 227, "server_ms": 34.5}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.46710601262748, "probs": [0.17244943152985545, 0.1836151222987057, 0.6439354461714388], "top": "permitted", "option_logits": null, "prompt_sha256": "cc59abf9dfd3346910e8d93dc78fe4cd21d0c20962370b801e609b82712c7ea7", "input_tokens": 228, "server_ms": 34.66}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.2394029982388, "probs": [0.11048645425133781, 0.06688603847853027, 0.8226275072701319], "top": "insufficient", "option_logits": null, "prompt_sha256": "45e75124dce70ee83cbbd2df43d7c5f3045b0da83559f8b4772e8cb6fa82dbb0", "input_tokens": 225, "server_ms": 34.28}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.82828897237778, "probs": [0.07600853330506088, 0.8779775900702179, 0.046013876624721124], "top": "permitted", "option_logits": null, "prompt_sha256": "902f5bf4dd52878d854ff660af610beb5cd00553cb6cf9fa5c1e7a5ea010391b", "input_tokens": 218, "server_ms": 34.98}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.289852974936366, "probs": [0.07014840077352269, 0.16883849753435962, 0.7610131016921177], "top": "B", "option_logits": null, "prompt_sha256": "d4a4f3c22ce6befbff78058a364e4617bda379057c95b5bacf2410a162ea27f7", "input_tokens": 221, "server_ms": 34.35}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.781668826937675, "probs": [0.7195969840802272, 0.2637194956142592, 0.016683520305513583], "top": "insufficient", "option_logits": null, "prompt_sha256": "fc0371a2d78013d514b134511d38b37de6e71a39ad2fa9b95b0119945db7a316", "input_tokens": 227, "server_ms": 35.07}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.24050219170749, "probs": [0.6328238128438389, 0.029254643239392513, 0.33792154391676854], "top": "A", "option_logits": null, "prompt_sha256": "5521ebbd2304bdf6049132465a2b8c7b28c862f462f50f98d90efeb483601e64", "input_tokens": 207, "server_ms": 34.32}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.88934997282922, "probs": [0.22351946247571317, 0.12708523693891682, 0.64939530058537], "top": "A", "option_logits": null, "prompt_sha256": "c11ad01ccd57e2b3b849c07b89887124412bce4a8e5506adaeb954f23c598ac7", "input_tokens": 197, "server_ms": 35.05}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.478354992344975, "probs": [0.038658346366163006, 0.013306059058578745, 0.9480355945752582], "top": "supported", "option_logits": null, "prompt_sha256": "8d5e868c2065409c5dcedc10c1d875c102dfbb48d2433b234f0ad0456f1049bc", "input_tokens": 213, "server_ms": 34.76}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.74564806371927, "probs": [0.01953747345276732, 0.9553520377742281, 0.025110488773004678], "top": "contradicted", "option_logits": null, "prompt_sha256": "6ac7738979b2b31bef2f7e2f3d8984b0b2b559ee71fa4d2b728e28884e3183b5", "input_tokens": 212, "server_ms": 35.04}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.58425707928836, "probs": [0.05100292527189731, 0.035003776957469146, 0.9139932977706335], "top": "contradicted", "option_logits": null, "prompt_sha256": "66c4b4c8b3dc52b623b51acf50a373bf625ad5559da7c3bf90c47f56e7c2829b", "input_tokens": 202, "server_ms": 34.76}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 38.00140111707151, "probs": [0.09842495451008554, 0.01699017293024806, 0.8845848725596664], "top": "insufficient", "option_logits": null, "prompt_sha256": "7db376c4d8c96d2c69fda861429911890ec2d70b0f375e60f65c846f1693e18a", "input_tokens": 203, "server_ms": 35.34}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.320202914997935, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "03449c2d41ee64f00d22b4b6e619aa69ce6d623d00b4972f503f5ed336bba08e", "input_tokens": 218, "server_ms": 34.65}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.70418814383447, "probs": [0.9331701885712249, 0.04890646327936172, 0.017923348149413328], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ed1d2ff423bdb9e622336508299a3180255754a84d1847b9b5ba5d1d5623692", "input_tokens": 221, "server_ms": 35.01}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.55894606001675, "probs": [0.05755147392994642, 0.0327217252711575, 0.9097268007988961], "top": "insufficient", "option_logits": null, "prompt_sha256": "fb0073f867c77a04f9287adbd88276517a2a85e5cf9dcfb04c81ff5ba768e03f", "input_tokens": 215, "server_ms": 34.88}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.70030802115798, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "insufficient", "option_logits": null, "prompt_sha256": "51c4ac799c31ceb752046516252cb8d59c3a3c1654c685c82476530a4d74a689", "input_tokens": 220, "server_ms": 34.89}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.32517408207059, "probs": [0.040967527214491276, 0.8323022347202245, 0.12673023806528433], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a5a9b08e7ca06d1f5808cdcd3270c79b064cd4fed16b4be278fc7d5f58a1d1f", "input_tokens": 206, "server_ms": 34.77}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.60075685568154, "probs": [0.048847636440916636, 0.07578275550521898, 0.8753696080538644], "top": "insufficient", "option_logits": null, "prompt_sha256": "555fd1e019506f048448ae23a6bac4fe7f3e9f2fe60181be02acfdf8225318e5", "input_tokens": 212, "server_ms": 34.91}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.640035850927234, "probs": [0.07319171247064139, 0.2264135372721482, 0.7003947502572104], "top": "insufficient", "option_logits": null, "prompt_sha256": "1fc713f5dfc86b0d90f75094b087d920fbfeb9830fb4eec74f78d2f1d67fbf66", "input_tokens": 203, "server_ms": 34.78}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.85038995556533, "probs": [0.9387496456243781, 0.055776151759141294, 0.005474202616480596], "top": "B", "option_logits": null, "prompt_sha256": "77e6633b56fd111f3cf38ff30d1c308c084b3de608efb9682cf02409dc31c9db", "input_tokens": 191, "server_ms": 35.15}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.610016064718366, "probs": [0.46903312718147117, 0.38856479455009935, 0.14240207826842943], "top": "supported", "option_logits": null, "prompt_sha256": "83104242d8900e7e51e6ca6f99d29588121dd7153c0fda9d487e3eafef91196e", "input_tokens": 224, "server_ms": 34.81}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.68548811785877, "probs": [0.7720397533404616, 0.11755438599864407, 0.11040586066089443], "top": "insufficient", "option_logits": null, "prompt_sha256": "66f0b52c1eee120f16069ed75fbd7bfc556b5a3dc09c27f809799b32ae74e959", "input_tokens": 221, "server_ms": 34.97}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.45078505016863, "probs": [0.014236196905258059, 0.6140385266223682, 0.3717252764723737], "top": "insufficient", "option_logits": null, "prompt_sha256": "fac76f29223ea6573fb84bb8cb70a27292765e7f4f90ebb19001e8a0b7bf0533", "input_tokens": 229, "server_ms": 34.67}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.786479108035564, "probs": [0.06291220787817095, 0.0030966605414949147, 0.9339911315803341], "top": "supported", "option_logits": null, "prompt_sha256": "89df7654ae643d961d5bd0a1cf472717b674fe14a8f5377a6918c1103303e14f", "input_tokens": 226, "server_ms": 35.15}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.55441587418318, "probs": [0.5216500126916775, 0.26161676918070537, 0.21673321812761712], "top": "prohibited", "option_logits": null, "prompt_sha256": "73a309dd6a1605e1d928e2a4947516f3ae54fda513d77cea9d4fdf892dbbebb5", "input_tokens": 226, "server_ms": 34.89}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.78924886137247, "probs": [0.843310768581531, 0.1455727910071266, 0.01111644041134243], "top": "prohibited", "option_logits": null, "prompt_sha256": "97510bceb98f70589de7351d3f810c2e4c175b03bbf6ddec55eca00ef9c199ee", "input_tokens": 215, "server_ms": 35.13}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.367114098742604, "probs": [0.42929779219364844, 0.486689574582582, 0.08401263322376958], "top": "permitted", "option_logits": null, "prompt_sha256": "5a1d1ca84275c476cd245cafcd0e1fd36598a6dae3c643b8ea9d60c5b57d6535", "input_tokens": 225, "server_ms": 34.57}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.872829008847475, "probs": [0.19560247361171754, 0.23611000151056785, 0.5682875248777147], "top": "permitted", "option_logits": null, "prompt_sha256": "fec6fd7a8c54ee7856b56b8cc85aa3ff4cb617820fd52f7faf204e2efb4a86ff", "input_tokens": 224, "server_ms": 35.09}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.19325293786824, "probs": [0.9323898466010938, 0.02450695094155103, 0.04310320245735524], "top": "B", "option_logits": null, "prompt_sha256": "fcc6cca56205c973fe5ee42de7bbca225406234b5346f63e55af7d8c5f115d9a", "input_tokens": 213, "server_ms": 34.43}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.4687360599637, "probs": [0.643833389199536, 0.3438005390976864, 0.012366071702777563], "top": "insufficient", "option_logits": null, "prompt_sha256": "176f3f577d5802518ba6bb8a36039f8c2c7fa4991aac2e236870e0189d0b31a6", "input_tokens": 218, "server_ms": 34.65}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.228543078526855, "probs": [0.020042268594912593, 0.21743089555343206, 0.7625268358516554], "top": "B", "option_logits": null, "prompt_sha256": "8494413c37184f76cf07859eef5fe11f7373b1acb0da7cbd005226a5bb00610d", "input_tokens": 209, "server_ms": 34.5}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.67507686279714, "probs": [0.33058297459294167, 0.050336110784327995, 0.6190809146227304], "top": "A", "option_logits": null, "prompt_sha256": "70f353d85ca31feb6cd6d5c421fe7a2e877a313657d44902068dc2f6ef451bf8", "input_tokens": 207, "server_ms": 34.96}], "seconds": 5.4, "failures": 0}, "multifield": {"rows": [{"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.345415990799665, "probs": [0.14720923592665844, 0.8527907640733415], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.345415990799665, "probs": [0.8214121479792841, 0.17115665028167817, 0.00743120173903778], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.345415990799665, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 46.345415990799665, "probs": [0.9305404571279347, 0.06673814075012655, 0.0027214021219388934], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.91643002256751, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.91643002256751, "probs": [0.8095965925134007, 0.07462670936879237, 0.11577669811780701], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.91643002256751, "probs": [0.9471470189565372, 0.052852981043462664], "top": "same"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.91643002256751, "probs": [0.9350414422025257, 0.05915293776423124, 0.005805620033243134], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.79053895920515, "probs": [0.22187782400103293, 0.778122175998967], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.79053895920515, "probs": [0.09874410354776683, 0.06776898998409085, 0.8334869064681424], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.79053895920515, "probs": [0.07967467169628852, 0.9203253283037116], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.79053895920515, "probs": [0.10841793826059654, 0.0843556864709907, 0.8072263752684127], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.88650895655155, "probs": [0.21123545993246395, 0.788764540067536], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.88650895655155, "probs": [0.11796301878836102, 0.8782943442297925, 0.0037426369818465326], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.88650895655155, "probs": [0.8527907640733415, 0.14720923592665844], "top": "same"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.88650895655155, "probs": [0.24350964936246988, 0.7532804299591004, 0.0032099206784296486], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.678607024252415, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.678607024252415, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.678607024252415, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.678607024252415, "probs": [0.9344875966425453, 0.06294564908924384, 0.002566754268210852], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.9256191495806, "probs": [0.7318060856600669, 0.2681939143399331], "top": "moved"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.9256191495806, "probs": [0.727805131100519, 0.25050783114702185, 0.02168703775245924], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.9256191495806, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.9256191495806, "probs": [0.8387192579214904, 0.15415437985672864, 0.007126362221781067], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.61421607993543, "probs": [0.10592235395880453, 0.8940776460411956], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.61421607993543, "probs": [0.928892622801621, 0.06661995929549502, 0.004487417902884053], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.61421607993543, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.61421607993543, "probs": [0.9588514552888884, 0.039099387288032916, 0.002049157423078645], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.16646289266646, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.16646289266646, "probs": [0.14648079183179633, 0.8485708719675852, 0.004948336200618522], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.16646289266646, "probs": [0.9248060506082927, 0.07519394939170732], "top": "same"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.16646289266646, "probs": [0.24334788832895243, 0.7527800186067107, 0.0038720930643368974], "top": "meet_wren"}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.01124387793243, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.01124387793243, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.01124387793243, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.01124387793243, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.0127539485693, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.0127539485693, "probs": [0.8089141677995819, 0.1910858322004182], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.0127539485693, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.0127539485693, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.14600688405335, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.14600688405335, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.14600688405335, "probs": [0.9439168356016766, 0.0560831643983235], "top": "same"}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.14600688405335, "probs": [0.9156019658234928, 0.08439803417650704], "top": "stay"}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.944823978468776, "probs": [0.3201997500236374, 0.6798002499763626], "top": "holds"}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.944823978468776, "probs": [0.12511350509772579, 0.8748864949022742], "top": "departure"}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.944823978468776, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.944823978468776, "probs": [0.13214384255458153, 0.8678561574454184], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.39912988245487, "probs": [0.25606104458540857, 0.7439389554145914], "top": "holds"}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.39912988245487, "probs": [0.08439803417650704, 0.9156019658234928], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.39912988245487, "probs": [0.11840617139815589, 0.8815938286018441], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.39912988245487, "probs": [0.10012611628383591, 0.8998738837161642], "top": "departure"}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.50527993403375, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.50527993403375, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.50527993403375, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.50527993403375, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.4635301399976, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.4635301399976, "probs": [0.8184242551251981, 0.18157574487480185], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.4635301399976, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.4635301399976, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.74048288911581, "probs": [0.12511350509772579, 0.8748864949022742], "top": "holds"}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.74048288911581, "probs": [0.9248060506082927, 0.07519394939170732], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.74048288911581, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.74048288911581, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.29107690602541, "probs": [0.7193152314525416, 0.2806847685474584], "top": "moved"}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.29107690602541, "probs": [0.08937422170979005, 0.91062577829021], "top": "outside"}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.29107690602541, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.29107690602541, "probs": [0.07519394939170732, 0.9248060506082927], "top": "outside"}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.77887386828661, "probs": [0.15526082656247686, 0.844739173437523], "top": "holds"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.77887386828661, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.77887386828661, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.77887386828661, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.545169876888394, "probs": [0.45308504361568214, 0.5469149563843178], "top": "holds"}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.545169876888394, "probs": [0.25606104458540857, 0.7439389554145914], "top": "outside"}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.545169876888394, "probs": [0.15526082656247686, 0.844739173437523], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.545169876888394, "probs": [0.43758935794935333, 0.5624106420506467], "top": "outside"}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.88217500410974, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.88217500410974, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.88217500410974, "probs": [0.9203253283037116, 0.07967467169628852], "top": "same"}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.88217500410974, "probs": [0.936892368084393, 0.06310763191560703], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.060685861855745, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.060685861855745, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.060685861855745, "probs": [0.9156019658234928, 0.08439803417650704], "top": "same"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.060685861855745, "probs": [0.91062577829021, 0.08937422170979005], "top": "stay"}], "seconds": 1.0, "failures": 0}}, "finished_utc": "2026-09-30T16:30:10Z"} \ No newline at end of file diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/vram.csv b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/vram.csv new file mode 100644 index 0000000..9c3835f --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r3/vram.csv @@ -0,0 +1,836 @@ +# pid=1884042 gpu=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff +1790785723.858692079,8812,8822 +1790785723.961606357,8812,8822 +1790785724.065286363,8812,8822 +1790785724.166397291,8812,8822 +1790785724.268539420,8812,8822 +1790785724.372357398,8812,8822 +1790785724.475378407,8812,8822 +1790785724.578682759,8812,8822 +1790785724.678638223,8812,8822 +1790785724.779488838,8812,8822 +1790785724.881270022,8812,8822 +1790785724.984505124,8812,8822 +1790785725.085237617,8812,8822 +1790785725.187184274,8812,8822 +1790785725.285596190,8812,8822 +1790785725.396265107,8812,8822 +1790785725.504412454,8812,8822 +1790785725.612396919,8812,8822 +1790785725.721188204,8812,8822 +1790785725.829428932,8812,8822 +1790785725.938797513,8812,8822 +1790785726.048575728,8812,8822 +1790785726.155808505,8812,8822 +1790785726.263311945,8812,8822 +1790785726.371124119,8812,8822 +1790785726.477372735,8812,8822 +1790785726.584720022,8812,8822 +1790785726.695117085,8812,8822 +1790785726.801581433,8812,8828 +1790785726.910742812,8818,8828 +1790785727.020747521,8818,8828 +1790785727.129719297,8818,8828 +1790785727.237926545,8818,8828 +1790785727.340327757,8818,8828 +1790785727.447922238,8818,8828 +1790785727.555892013,8818,8828 +1790785727.660513290,8818,8828 +1790785727.764923195,8818,8828 +1790785727.867172615,8818,8828 +1790785727.973338230,8818,8828 +1790785728.079363223,8818,8828 +1790785728.181624823,8818,8828 +1790785728.285348690,8818,8828 +1790785728.387720562,8818,8828 +1790785728.493660104,8818,8828 +1790785728.595346278,8818,8828 +1790785728.703689457,8818,8828 +1790785728.804240038,8818,8828 +1790785728.911757718,8818,8828 +1790785729.017235905,8818,8828 +1790785729.122403228,8818,8828 +1790785729.226147216,8818,8828 +1790785729.328916852,8818,8828 +1790785729.435276279,8818,8828 +1790785729.541673356,8818,8828 +1790785729.645036169,8818,8828 +1790785729.752948954,8834,8844 +1790785729.857834924,8834,8844 +1790785729.962109367,8834,8844 +1790785730.065860834,8834,8844 +1790785730.169347418,8834,8844 +1790785730.273257187,8834,8844 +1790785730.376605460,8834,8844 +1790785730.479593369,8834,8844 +1790785730.579761616,8834,8844 +1790785730.681339128,8834,8844 +1790785730.787413772,8834,8844 +1790785730.890413541,8834,8844 +1790785730.993756153,8834,8844 +1790785731.095318186,8834,8844 +1790785731.194303529,8834,8844 +1790785731.296077584,8834,8844 +1790785731.395550743,8834,8844 +1790785731.500516184,8834,8844 +1790785731.603096458,8834,8844 +1790785731.701797528,8834,8844 +1790785731.809208376,8834,8844 +1790785731.911788891,8834,8844 +1790785732.014413965,8834,8844 +1790785732.116272871,8834,8844 +1790785732.217508010,8834,8844 +1790785732.321278937,8834,8844 +1790785732.425261957,8834,8844 +1790785732.525758718,8834,8844 +1790785732.629090730,8834,8844 +1790785732.735027912,8834,8844 +1790785732.834448300,8834,8844 +1790785732.938029726,8834,8844 +1790785733.041838604,8834,8844 +1790785733.148846258,8834,8844 +1790785733.253815509,8834,8844 +1790785733.354929267,8834,8844 +1790785733.461254943,8834,8844 +1790785733.561304338,8834,8844 +1790785733.661309163,8834,8844 +1790785733.761037425,8834,8844 +1790785733.865404819,8834,8844 +1790785733.969953066,8834,8844 +1790785734.073203837,8834,8844 +1790785734.176670611,8834,8844 +1790785734.281627282,8834,8844 +1790785734.383536368,8834,8844 +1790785734.487086793,8834,8844 +1790785734.587973828,8834,8844 +1790785734.696641072,8834,8844 +1790785734.801396980,8834,8844 +1790785734.903261876,8834,8844 +1790785735.005494656,8834,8844 +1790785735.109559999,8834,8844 +1790785735.206454079,8834,8844 +1790785735.303736582,8834,8844 +1790785735.405654728,8834,8844 +1790785735.509406765,8834,8844 +1790785735.611780438,8834,8844 +1790785735.711420878,8834,8844 +1790785735.816822574,8834,8844 +1790785735.918229185,8834,8844 +1790785736.017915066,8834,8844 +1790785736.119276556,8834,8844 +1790785736.231512590,8834,8844 +1790785736.332481726,8834,8844 +1790785736.434686906,8834,8844 +1790785736.537401732,8834,8844 +1790785736.637105325,8834,8844 +1790785736.740514988,8834,8844 +1790785736.841519184,8834,8844 +1790785736.943030296,8834,8844 +1790785737.042195702,8834,8844 +1790785737.145507434,8834,8844 +1790785737.247761974,8834,8844 +1790785737.353285132,8834,8844 +1790785737.456511523,8834,8844 +1790785737.556092443,8834,8844 +1790785737.661134465,8834,8844 +1790785737.764045653,8834,8844 +1790785737.867819770,8834,8844 +1790785737.973536120,8834,8844 +1790785738.077123485,8834,8844 +1790785738.181288067,8834,8844 +1790785738.282024461,8834,8844 +1790785738.383603933,8834,8844 +1790785738.488061468,8834,8844 +1790785738.589766822,8834,8844 +1790785738.693346468,8834,8844 +1790785738.795655119,8834,8844 +1790785738.901751932,8834,8844 +1790785739.007519933,8834,8844 +1790785739.109026955,8834,8844 +1790785739.208828037,8834,8844 +1790785739.312355372,8834,8844 +1790785739.415365921,8834,8844 +1790785739.514984921,8834,8844 +1790785739.618507866,8834,8844 +1790785739.721508725,8834,8844 +1790785739.820053403,8834,8844 +1790785739.925804753,8834,8844 +1790785740.029830304,8834,8844 +1790785740.131189254,8834,8844 +1790785740.227674088,8834,8844 +1790785740.328590824,8834,8844 +1790785740.431706424,8834,8844 +1790785740.532390836,8834,8844 +1790785740.633979639,8834,8844 +1790785740.738006939,8834,8844 +1790785740.839191828,8834,8844 +1790785740.941361267,8834,8844 +1790785741.043087601,8834,8844 +1790785741.149930763,8834,8844 +1790785741.253286727,8834,8844 +1790785741.357632371,8834,8844 +1790785741.463082447,8834,8844 +1790785741.563521537,8834,8864 +1790785741.663943186,8854,8864 +1790785741.768343221,8854,8864 +1790785741.872076828,8854,8864 +1790785741.980627700,8854,8864 +1790785742.083453607,8854,8864 +1790785742.187604988,8854,8864 +1790785742.290053161,8854,8864 +1790785742.395000372,8854,8864 +1790785742.496082039,8854,8864 +1790785742.599053867,8854,8864 +1790785742.703519963,8854,8864 +1790785742.807549333,8854,8864 +1790785742.911074538,8854,8864 +1790785743.014708354,8854,8864 +1790785743.115943822,8854,8864 +1790785743.219402047,8854,8864 +1790785743.333750364,8854,8864 +1790785743.435635370,8854,8864 +1790785743.538135014,8854,8864 +1790785743.637904906,8854,8864 +1790785743.741257608,8854,8864 +1790785743.841350434,8854,8864 +1790785743.943729366,8854,8864 +1790785744.045432290,8854,8864 +1790785744.144063329,8854,8864 +1790785744.244824422,8854,8864 +1790785744.351657175,8854,8864 +1790785744.453499230,8854,8864 +1790785744.554008821,8854,8864 +1790785744.660862054,8854,8864 +1790785744.771660771,8854,8864 +1790785744.881315645,8854,8864 +1790785744.991075990,8854,8864 +1790785745.099209207,8854,8864 +1790785745.210214827,8854,8864 +1790785745.319694409,8854,8864 +1790785745.433696303,8854,8864 +1790785745.540977230,8854,8864 +1790785745.652077131,8854,8864 +1790785745.763094321,8854,8864 +1790785745.873012668,8854,8864 +1790785745.981867503,8854,8864 +1790785746.091798141,8854,8864 +1790785746.204367388,8854,8864 +1790785746.313383755,8854,8864 +1790785746.414960028,8854,8864 +1790785746.519166490,8854,8864 +1790785746.625544447,8854,8864 +1790785746.732131317,8854,8864 +1790785746.837916987,8854,8864 +1790785746.939693152,8854,8864 +1790785747.043637252,8854,8864 +1790785747.149462472,8854,8864 +1790785747.253653704,8854,8864 +1790785747.358158681,8854,8864 +1790785747.461101399,8854,8864 +1790785747.565784737,8854,8864 +1790785747.669728136,8854,8864 +1790785747.770756552,8854,8864 +1790785747.872230894,8854,8864 +1790785747.976611888,8854,8864 +1790785748.083962377,8854,8864 +1790785748.188936058,8854,8864 +1790785748.291256449,8854,8864 +1790785748.393817453,8854,8864 +1790785748.497567680,8854,8864 +1790785748.601156095,8854,8864 +1790785748.705573780,8854,8864 +1790785748.807350785,8854,8864 +1790785748.913617441,8854,8864 +1790785749.016797012,8854,8864 +1790785749.121207137,8854,8864 +1790785749.224160525,8854,8864 +1790785749.321563090,8854,8864 +1790785749.426888265,8854,8864 +1790785749.526537236,8854,8864 +1790785749.627218848,8854,8864 +1790785749.733349833,8854,8864 +1790785749.838872390,8854,8864 +1790785749.940981709,8854,8864 +1790785750.045165611,8854,8864 +1790785750.149768618,8854,8864 +1790785750.251134308,8854,8864 +1790785750.360808693,8854,8864 +1790785750.462961882,8854,8864 +1790785750.569622052,8854,8864 +1790785750.671992524,8854,8864 +1790785750.773072941,8854,8864 +1790785750.877636087,8854,8864 +1790785750.979295401,8854,8864 +1790785751.086396636,8854,8864 +1790785751.191217186,8854,8864 +1790785751.291712736,8854,8864 +1790785751.397410566,8854,8864 +1790785751.501929441,8854,8864 +1790785751.606175004,8854,8864 +1790785751.704668632,8854,8864 +1790785751.809020166,8854,8864 +1790785751.913703134,8854,8864 +1790785752.021802121,8854,8864 +1790785752.132026781,8854,8864 +1790785752.236857481,8854,8864 +1790785752.341323456,8854,8864 +1790785752.446109195,8854,8864 +1790785752.549462508,8854,8864 +1790785752.653131875,8854,8864 +1790785752.757935534,8854,8864 +1790785752.859764319,8854,8864 +1790785752.967523892,8854,8864 +1790785753.068233155,8854,8864 +1790785753.173450339,8854,8864 +1790785753.279059737,8854,8864 +1790785753.379270914,8854,8864 +1790785753.477111545,8854,8864 +1790785753.579108272,8854,8864 +1790785753.681377433,8854,8864 +1790785753.782222117,8854,8864 +1790785753.885269696,8854,8864 +1790785753.989509169,8854,8864 +1790785754.093832593,8854,8864 +1790785754.198048365,8854,8864 +1790785754.301799683,8854,8864 +1790785754.414075907,8854,8864 +1790785754.522658899,8854,8864 +1790785754.627091254,8854,8864 +1790785754.731718311,8854,8864 +1790785754.845497452,8854,8864 +1790785754.953647059,8854,8864 +1790785755.062647636,8854,8864 +1790785755.165660645,8854,8864 +1790785755.265830402,8854,8864 +1790785755.366296772,8854,8864 +1790785755.465762050,8854,8864 +1790785755.572165978,8854,8864 +1790785755.677432272,8854,8864 +1790785755.780016126,8854,8864 +1790785755.886087600,8854,8864 +1790785755.989772387,8854,8864 +1790785756.095680358,8854,8864 +1790785756.202029275,8854,8864 +1790785756.314111047,8854,8864 +1790785756.421675018,8854,8864 +1790785756.529199697,8854,8864 +1790785756.635495934,8854,8864 +1790785756.740075901,8854,8864 +1790785756.845265394,8854,8864 +1790785756.942973353,8854,8864 +1790785757.044270293,8854,8864 +1790785757.148364734,8854,8864 +1790785757.253888541,8854,8864 +1790785757.357019661,8854,8864 +1790785757.460988031,8854,8864 +1790785757.563457354,8854,8864 +1790785757.665645183,8854,8864 +1790785757.763666856,8854,8864 +1790785757.866981428,8854,8864 +1790785757.972518236,8854,8864 +1790785758.078132624,8854,8864 +1790785758.179037169,8854,8864 +1790785758.281742564,8854,8864 +1790785758.379803777,8854,8864 +1790785758.483999580,8854,8864 +1790785758.589735129,8854,8864 +1790785758.693579218,8854,8864 +1790785758.793109767,8854,8864 +1790785758.891426963,8854,8864 +1790785758.993114217,8854,8864 +1790785759.097585222,8854,8864 +1790785759.197553527,8854,8864 +1790785759.301555067,8854,8864 +1790785759.402041617,8854,8864 +1790785759.505974636,8854,8864 +1790785759.605923470,8854,8864 +1790785759.709589647,8854,8864 +1790785759.808157955,8854,8864 +1790785759.913682803,8854,8864 +1790785760.015734261,8854,8864 +1790785760.118162923,8854,8864 +1790785760.216369077,8854,8864 +1790785760.321908585,8854,8864 +1790785760.425663512,8854,8864 +1790785760.523713795,8854,8864 +1790785760.626848305,8854,8864 +1790785760.733641887,8854,8864 +1790785760.840230806,8854,8864 +1790785760.944335597,8854,8864 +1790785761.048754513,8854,8864 +1790785761.154148689,8854,8864 +1790785761.256116366,8854,8864 +1790785761.358108143,8854,8864 +1790785761.462769601,8854,8864 +1790785761.568085465,8854,8864 +1790785761.673605423,8854,8864 +1790785761.778150929,8854,8864 +1790785761.881950397,8854,8864 +1790785761.985868976,8854,8864 +1790785762.090426523,8854,8864 +1790785762.192063286,8854,8864 +1790785762.293461077,8854,8864 +1790785762.400459371,8854,8864 +1790785762.502150605,8854,8864 +1790785762.607781373,8854,8864 +1790785762.708364565,8854,8864 +1790785762.809875817,8854,8864 +1790785762.911125826,8854,8864 +1790785763.012255323,8854,8864 +1790785763.117095963,8854,8864 +1790785763.220073071,8854,8864 +1790785763.324247223,8854,8864 +1790785763.427044750,8854,8864 +1790785763.533673990,8854,8864 +1790785763.637561469,8854,8864 +1790785763.739403004,8854,8864 +1790785763.839327748,8854,8864 +1790785763.939184791,8854,8864 +1790785764.036663807,8854,8864 +1790785764.142073634,8854,8864 +1790785764.239607560,8854,8864 +1790785764.345593663,8854,8864 +1790785764.447085855,8854,8864 +1790785764.549587158,8854,8864 +1790785764.653984772,8854,8864 +1790785764.757294375,8854,8864 +1790785764.861643539,8854,8864 +1790785764.964085881,8854,8864 +1790785765.070968084,8854,8864 +1790785765.176367270,8854,8864 +1790785765.280191828,8854,8864 +1790785765.385307081,8854,8864 +1790785765.486290357,8854,8864 +1790785765.590116555,8854,8864 +1790785765.692478957,8854,8864 +1790785765.795107321,8854,8864 +1790785765.899431165,8854,8864 +1790785765.998066264,8854,8864 +1790785766.102584990,8854,8864 +1790785766.209053788,8854,8864 +1790785766.309220055,8854,8864 +1790785766.414173066,8854,8864 +1790785766.517766082,8854,8864 +1790785766.621693901,8854,8864 +1790785766.725493588,8854,8864 +1790785766.833335342,8854,8864 +1790785766.934902545,8854,8864 +1790785767.041502924,8854,8864 +1790785767.141161425,8854,8864 +1790785767.244650070,8854,8864 +1790785767.344297031,8854,8864 +1790785767.445036233,8854,8864 +1790785767.550967856,8854,8864 +1790785767.653359528,8854,8864 +1790785767.757880063,8854,8864 +1790785767.860087933,8854,8864 +1790785767.967772985,8854,8864 +1790785768.071574793,8854,8864 +1790785768.175411331,8854,8864 +1790785768.280184360,8854,8864 +1790785768.387643610,8854,8864 +1790785768.491430787,8854,8864 +1790785768.601199703,8854,8864 +1790785768.709825965,8854,8864 +1790785768.817486927,8854,8864 +1790785768.924921526,8854,8864 +1790785769.033603080,8854,8864 +1790785769.141798137,8854,8864 +1790785769.250653833,8854,8864 +1790785769.359000892,8854,8864 +1790785769.468096650,8854,8864 +1790785769.577299309,8854,8864 +1790785769.688825255,8854,8864 +1790785769.794793317,8854,8864 +1790785769.901133904,8854,8864 +1790785770.007901465,8854,8864 +1790785770.113465703,8854,8864 +1790785770.217986389,8854,8864 +1790785770.322660507,8854,8864 +1790785770.431452801,8854,8864 +1790785770.532212835,8854,8864 +1790785770.638978436,8854,8864 +1790785770.743536883,8854,8864 +1790785770.842853470,8854,8864 +1790785770.942076876,8854,8864 +1790785771.045738912,8854,8864 +1790785771.150275618,8854,8864 +1790785771.253645581,8854,8864 +1790785771.356975124,8854,8864 +1790785771.462068556,8854,8864 +1790785771.564618480,8854,8864 +1790785771.667494327,8854,8864 +1790785771.771710220,8854,8864 +1790785771.874047741,8854,8864 +1790785771.977007119,8854,8864 +1790785772.079137949,8854,8864 +1790785772.181829494,8854,8864 +1790785772.281364053,8854,8864 +1790785772.378904130,8854,8864 +1790785772.475399845,8854,8864 +1790785772.577275711,8854,8864 +1790785772.681438123,8854,8864 +1790785772.783256238,8854,8864 +1790785772.884909672,8854,8864 +1790785772.987268943,8854,8864 +1790785773.092166383,8854,8864 +1790785773.197590610,8854,8864 +1790785773.300345816,8854,8864 +1790785773.403380895,8854,8864 +1790785773.504971888,8854,8864 +1790785773.612239125,8854,8864 +1790785773.718090496,8854,8864 +1790785773.823224669,8854,8864 +1790785773.928314362,8854,8864 +1790785774.036705902,8854,8864 +1790785774.138943642,8854,8864 +1790785774.241906740,8854,8864 +1790785774.345143872,8854,8864 +1790785774.449724268,8854,8864 +1790785774.549863705,8854,8864 +1790785774.653810714,8854,8864 +1790785774.754325565,8854,8864 +1790785774.854136587,8854,8864 +1790785774.957884355,8854,8864 +1790785775.061921725,8854,8864 +1790785775.165874014,8854,8864 +1790785775.271539164,8854,8864 +1790785775.374999578,8854,8864 +1790785775.476980665,8854,8864 +1790785775.577866149,8854,8864 +1790785775.682613718,8854,8864 +1790785775.781831944,8854,8864 +1790785775.883769591,8854,8864 +1790785775.985555976,8854,8864 +1790785776.087263280,8854,8864 +1790785776.189709063,8854,8864 +1790785776.295583714,8854,8864 +1790785776.401399455,8854,8864 +1790785776.505508506,8854,8864 +1790785776.605217537,8854,8864 +1790785776.708908124,8854,8864 +1790785776.815874008,8854,8864 +1790785776.920822088,8854,8864 +1790785777.026089753,8854,8864 +1790785777.133129538,8854,8864 +1790785777.232476285,8854,8864 +1790785777.332045825,8854,8864 +1790785777.433616968,8854,8864 +1790785777.535128899,8854,8864 +1790785777.638778475,8854,8864 +1790785777.743942939,8854,8864 +1790785777.847201101,8854,8864 +1790785777.949602122,8854,8864 +1790785778.055796878,8854,8864 +1790785778.160786269,8854,8864 +1790785778.263584895,8854,8864 +1790785778.364431240,8854,8864 +1790785778.471856439,8854,8864 +1790785778.577165234,8854,8864 +1790785778.681256685,8854,8864 +1790785778.792223554,8854,8864 +1790785778.901707377,8854,8864 +1790785779.011377281,8854,8864 +1790785779.121062706,8854,8864 +1790785779.229571067,8854,8864 +1790785779.339763757,8854,8864 +1790785779.451078650,8854,8864 +1790785779.563007060,8854,8864 +1790785779.673004369,8854,8864 +1790785779.781089695,8854,8864 +1790785779.888688686,8854,8864 +1790785779.996540210,8854,8864 +1790785780.108041665,8854,8864 +1790785780.217112653,8854,8864 +1790785780.327800649,8854,8864 +1790785780.437282881,8854,8864 +1790785780.546554361,8854,8864 +1790785780.649902794,8854,8864 +1790785780.753570250,8854,8864 +1790785780.855371935,8854,8864 +1790785780.954872194,8854,8864 +1790785781.053859378,8854,8864 +1790785781.165076330,8854,8864 +1790785781.273215597,8854,8864 +1790785781.382416856,8854,8864 +1790785781.486734250,8854,8864 +1790785781.586691754,8854,8864 +1790785781.689331679,8854,8864 +1790785781.797196622,8854,8864 +1790785781.899327431,8854,8864 +1790785781.998352295,8854,8864 +1790785782.101431625,8854,8864 +1790785782.201680322,8854,8864 +1790785782.302480046,8854,8864 +1790785782.417429771,8854,8864 +1790785782.527909474,8854,8864 +1790785782.636843121,8854,8864 +1790785782.745771346,8854,8864 +1790785782.856692615,8854,8864 +1790785782.970332485,8854,8864 +1790785783.076974005,8854,8864 +1790785783.185584877,8854,8864 +1790785783.293112297,8854,8864 +1790785783.402359947,8854,8864 +1790785783.513479758,8854,8864 +1790785783.613437352,8854,8864 +1790785783.717113689,8854,8864 +1790785783.819612722,8854,8864 +1790785783.923040245,8854,8864 +1790785784.019678092,8854,8864 +1790785784.122114094,8854,8864 +1790785784.224611698,8854,8864 +1790785784.329191384,8854,8864 +1790785784.429309640,8854,8864 +1790785784.531120406,8854,8864 +1790785784.635842794,8854,8864 +1790785784.747880425,8854,8864 +1790785784.856744951,8854,8864 +1790785784.964251371,8854,8864 +1790785785.067229269,8854,8864 +1790785785.172301851,8854,8864 +1790785785.273965295,8854,8864 +1790785785.377163576,8854,8864 +1790785785.477837528,8854,8864 +1790785785.580260160,8854,8864 +1790785785.688905863,8854,8864 +1790785785.793272007,8854,8864 +1790785785.895601438,8854,8864 +1790785785.997006219,8854,8864 +1790785786.105632052,8854,8864 +1790785786.209014565,8854,8864 +1790785786.312651581,8854,8864 +1790785786.419567424,8854,8864 +1790785786.522670014,8854,8864 +1790785786.633112568,8854,8864 +1790785786.733842910,8854,8864 +1790785786.839026234,8854,8864 +1790785786.941424326,8854,8864 +1790785787.046955093,8854,8864 +1790785787.150696511,8854,8864 +1790785787.254328737,8854,8864 +1790785787.357035712,8854,8864 +1790785787.459596966,8854,8864 +1790785787.562256411,8854,8864 +1790785787.663671152,8854,8864 +1790785787.766342247,8854,8864 +1790785787.871524660,8854,8864 +1790785787.974948224,8854,8864 +1790785788.081403652,8854,8864 +1790785788.185938848,8854,8864 +1790785788.290101920,8854,8864 +1790785788.394944080,8854,8864 +1790785788.498111840,8854,8864 +1790785788.601627425,8854,8864 +1790785788.707575377,8854,8864 +1790785788.810192362,8854,8864 +1790785788.909572790,8854,8864 +1790785789.009610374,8854,8864 +1790785789.112934137,8854,8864 +1790785789.215740823,8854,8864 +1790785789.318843194,8854,8864 +1790785789.421348357,8854,8864 +1790785789.524236234,8854,8864 +1790785789.627982122,8854,8864 +1790785789.729593354,8854,8864 +1790785789.833254520,8854,8864 +1790785789.933418187,8854,8864 +1790785790.035837879,8854,8864 +1790785790.139428975,8854,8864 +1790785790.243306654,8854,8864 +1790785790.347613758,8854,8864 +1790785790.453964044,8854,8864 +1790785790.560434032,8854,8864 +1790785790.667067052,8854,8864 +1790785790.771732950,8854,8864 +1790785790.876908853,8854,8864 +1790785790.983515453,8854,8864 +1790785791.086688434,8854,8864 +1790785791.183917147,8854,8864 +1790785791.289101801,8854,8864 +1790785791.393435085,8854,8864 +1790785791.494587562,8854,8864 +1790785791.593876889,8854,8864 +1790785791.697883109,8854,8864 +1790785791.799623384,8854,8864 +1790785791.902928196,8854,8864 +1790785792.008306292,8854,8864 +1790785792.113285833,8854,8864 +1790785792.216378773,8854,8864 +1790785792.321494106,8854,8864 +1790785792.425146212,8854,8864 +1790785792.526104567,8854,8864 +1790785792.629818904,8854,8864 +1790785792.735151220,8854,8864 +1790785792.837674933,8854,8864 +1790785792.941500611,8854,8864 +1790785793.045935886,8854,8864 +1790785793.148425889,8854,8864 +1790785793.253325129,8854,8864 +1790785793.355427928,8854,8864 +1790785793.461670864,8854,8864 +1790785793.561441366,8854,8864 +1790785793.664556036,8854,8864 +1790785793.769720569,8854,8864 +1790785793.871273871,8854,8864 +1790785793.975172130,8854,8864 +1790785794.079813768,8854,8864 +1790785794.186767181,8854,8864 +1790785794.291815513,8854,8864 +1790785794.396904516,8854,8864 +1790785794.501296161,8854,8864 +1790785794.605643375,8854,8864 +1790785794.707782114,8854,8864 +1790785794.813645185,8854,8864 +1790785794.915422030,8854,8864 +1790785795.016541377,8854,8864 +1790785795.115474750,8854,8864 +1790785795.214800547,8854,8864 +1790785795.321428707,8854,8864 +1790785795.425467818,8854,8864 +1790785795.528333355,8854,8864 +1790785795.633511008,8854,8864 +1790785795.738153856,8854,8864 +1790785795.840402606,8854,8864 +1790785795.944695449,8854,8864 +1790785796.049916964,8854,8864 +1790785796.153722591,8854,8864 +1790785796.256378356,8854,8864 +1790785796.360535548,8854,8864 +1790785796.461949819,8854,8864 +1790785796.565828508,8854,8864 +1790785796.669774157,8854,8864 +1790785796.774628867,8854,8864 +1790785796.877967419,8854,8864 +1790785796.982681138,8854,8864 +1790785797.080969113,8854,8864 +1790785797.186670623,8854,8864 +1790785797.286906850,8854,8864 +1790785797.386989265,8854,8864 +1790785797.489052934,8854,8864 +1790785797.594946405,8854,8864 +1790785797.697729202,8854,8864 +1790785797.801374098,8854,8864 +1790785797.901392253,8854,8864 +1790785798.000566938,8854,8864 +1790785798.103251243,8854,8864 +1790785798.204016637,8854,8864 +1790785798.304803841,8854,8864 +1790785798.406838378,8854,8864 +1790785798.508225938,8854,8864 +1790785798.612286070,8854,8864 +1790785798.716293040,8854,8864 +1790785798.822062340,8854,8864 +1790785798.925475563,8854,8864 +1790785799.029063859,8854,8864 +1790785799.127533546,8854,8864 +1790785799.231435735,8854,8864 +1790785799.333168510,8854,8864 +1790785799.434371838,8854,8864 +1790785799.536354915,8854,8864 +1790785799.640803251,8854,8864 +1790785799.747404330,8854,8864 +1790785799.848915682,8854,8864 +1790785799.950886449,8854,8864 +1790785800.048074002,8854,8864 +1790785800.150338622,8854,8864 +1790785800.251867104,8854,8864 +1790785800.357635995,8854,8864 +1790785800.454758947,8854,8864 +1790785800.555221607,8854,8864 +1790785800.653954177,8854,8864 +1790785800.760556977,8854,8864 +1790785800.865887642,8854,8864 +1790785800.967823008,8854,8864 +1790785801.070773767,8854,8864 +1790785801.172152757,8854,8864 +1790785801.275930775,8854,8864 +1790785801.379753873,8854,8864 +1790785801.479645786,8854,8864 +1790785801.586627570,8854,8864 +1790785801.691212847,8854,8864 +1790785801.788315449,8854,8864 +1790785801.893270210,8854,8864 +1790785801.995210187,8854,8864 +1790785802.096596617,8854,8864 +1790785802.201409956,8854,8864 +1790785802.303695157,8854,8864 +1790785802.404242708,8854,8864 +1790785802.508118327,8854,8864 +1790785802.608241003,8854,8864 +1790785802.709059337,8854,8864 +1790785802.809891601,8854,8864 +1790785802.911203861,8854,8864 +1790785803.009957261,8854,8864 +1790785803.110637894,8854,8864 +1790785803.214041787,8854,8864 +1790785803.313759978,8854,8864 +1790785803.415874148,8854,8864 +1790785803.516221226,8854,8864 +1790785803.621301058,8854,8864 +1790785803.720580925,8854,8864 +1790785803.824132630,8854,8864 +1790785803.934252740,8854,8864 +1790785804.040235583,8854,8864 +1790785804.141737864,8854,8864 +1790785804.246111968,8854,8864 +1790785804.348713753,8854,8864 +1790785804.449293564,8854,8864 +1790785804.554763801,8854,8864 +1790785804.659599830,8854,8864 +1790785804.762047863,8854,8864 +1790785804.866100394,8854,8864 +1790785804.971518260,8854,8864 +1790785805.076017226,8854,8864 +1790785805.177629249,8854,8864 +1790785805.273964092,8854,8864 +1790785805.377860871,8854,8864 +1790785805.479270862,8854,8864 +1790785805.581680704,8854,8864 +1790785805.684532600,8854,8864 +1790785805.789389521,8854,8864 +1790785805.895880489,8854,8864 +1790785805.998373392,8854,8864 +1790785806.102328392,8854,8864 +1790785806.205336190,8854,8864 +1790785806.304791169,8854,8864 +1790785806.409974573,8854,8864 +1790785806.515714493,8854,8864 +1790785806.619804254,8854,8864 +1790785806.721290635,8854,8864 +1790785806.825530078,8854,8864 +1790785806.924255098,8854,8864 +1790785807.027909014,8854,8864 +1790785807.128465196,8854,8864 +1790785807.229331180,8854,8864 +1790785807.327455124,8854,8864 +1790785807.434892283,8854,8864 +1790785807.532340208,8854,8864 +1790785807.635401378,8854,8864 +1790785807.740775823,8854,8864 +1790785807.841639598,8854,8864 +1790785807.945625848,8854,8864 +1790785808.049221004,8854,8864 +1790785808.153203203,8854,8864 +1790785808.254071798,8854,8864 +1790785808.354511168,8854,8864 +1790785808.460072826,8854,8864 +1790785808.563065884,8854,8864 +1790785808.666770261,8854,8864 +1790785808.772126477,8854,8864 +1790785808.875303367,8854,8864 +1790785808.976328884,8854,8864 +1790785809.078520574,8854,8864 +1790785809.181996708,8854,8864 +1790785809.289540768,8854,8864 +1790785809.392565837,8854,8864 +1790785809.498381558,8928,8938 +1790785809.601320596,8928,8938 +1790785809.704796090,8928,8938 +1790785809.807168462,8928,8938 +1790785809.909928068,8928,8938 +1790785810.013840377,8928,8938 +1790785810.121158305,8928,8938 +1790785810.224119473,8928,8938 +1790785810.329907644,8928,8938 +1790785810.432486378,8928,8938 diff --git a/services/intern-decision-serve/acceptance/gpu3-2026-09-30/run.sh b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/run.sh new file mode 100755 index 0000000..341b296 --- /dev/null +++ b/services/intern-decision-serve/acceptance/gpu3-2026-09-30/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# run.sh