diff --git a/scripts/intern-decision-warmup b/scripts/intern-decision-warmup new file mode 100755 index 0000000..b70090e --- /dev/null +++ b/scripts/intern-decision-warmup @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Warm every length bucket of intern-decision's kernel autotune cache. + +The fast kernels (triton + fla) autotune per shape BUCKET: the first call that lands in a new +bucket costs seconds of autotune. The cache lives in the container's `triton-cache` volume, so +it survives ordinary recreates — this script is for AFTER AN IMAGE CHANGE (a new triton/fla +version keys new entries). Idempotent: a second run is all fast. + +One noul /v1/systemone call per bucket, smallest up to the service's own MAX_TOKENS (read from +/health; never hard-coded). Aims just under each bucket's top edge; the aim is calibrated from +the service's own reported input_tokens after every call, so drift cannot leave a bucket cold. +Prints each call's bucket, the model's own token count, and the wall time. Exits non-zero if +any call fails or lands outside its bucket. + + scripts/intern-decision-warmup [--endpoint URL] [--bucket-width N] +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +import urllib.error +import urllib.request + +UNIT = "the quick brown fox. " # ~5.13 tokens against this service (measured 2026-09-30) + + +def post(endpoint: str, token: str, body: dict, timeout: float = 180.0): + req = urllib.request.Request( + endpoint.rstrip("/") + "/v1/systemone", + data=json.dumps(body).encode(), method="POST", + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + t0 = time.perf_counter() + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read()), time.perf_counter() - t0 + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b"{}"), time.perf_counter() - t0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--endpoint", default="http://intern-decision.fv.internal:8033") + ap.add_argument("--bucket-width", type=int, default=2048, + help="asserted bucket width; the acceptance run proves or corrects it") + args = ap.parse_args() + + token = subprocess.run(["secret", "get", "intern-decision/api-token"], + capture_output=True, text=True).stdout.strip() + if not token: + print("no token: secret get intern-decision/api-token", file=sys.stderr) + return 2 + health = json.load(urllib.request.urlopen(args.endpoint.rstrip("/") + "/health", timeout=15)) + max_tokens = int(health["max_tokens"]) + width = args.bucket_width + buckets = list(range(width, max_tokens + 1, width)) + print(f"endpoint={args.endpoint} max_tokens={max_tokens} bucket_width={width} " + f"buckets={len(buckets)}") + + def ask(reps: int): + return post(args.endpoint, token, {"state": UNIT * reps, + "questions": {"warm": {"type": "noul", + "instructions": "Is the system healthy?"}}}) + + # Two-point calibration of the tokenizer's linear model, input_tokens = a*reps + b. A single + # probe cannot separate a from b and its "calibration" overcorrects on the next aim (measured: + # the aim oscillates ±500 tokens around the bucket edge). Two probes 900 reps apart fit exactly. + s1, o1, _ = ask(100) + s2, o2, _ = ask(1000) + if s1 != 200 or s2 != 200: + print("calibration failed", s1, s2, file=sys.stderr) + return 2 + t1 = o1["usage"]["input_tokens"]; t2 = o2["usage"]["input_tokens"] + a = (t2 - t1) / 900.0 + b = t1 - a * 100 + print(f"calibrated: input_tokens = {a:.4f}*reps + {b:.1f}") + + failed, total = False, 0.0 + print(f"{'bucket<=':>9} {'tokens':>7} {'wall_s':>7} note") + for hi in buckets: + note = "" + got = status = 0 + wall = " n/a" + for attempt in (1, 2): # a second aim fixes a rounding miss; never miss a bucket + reps = max(1, round((hi - 60 - b) / a)) + status, out, w = ask(reps) + total += w + wall = f"{w:.2f}" + got = (out.get("usage") or {}).get("input_tokens", 0) + if status != 200 or hi - width < got <= hi: + break + if status != 200: + note = f"FAILED {status}: {(out.get('error') or {}).get('message', '')[:60]}" + failed = True + elif not (hi - width < got <= hi): + note = f"OUT OF BUCKET ({hi-width} < {got} <= {hi})" + failed = True + print(f"{hi:>9} {got:>7} {wall:>7} {note}") + print(f"total {total:.1f} s over {len(buckets)} buckets") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/intern-decision-serve/Dockerfile b/services/intern-decision-serve/Dockerfile index a88258d..e57a17d 100644 --- a/services/intern-decision-serve/Dockerfile +++ b/services/intern-decision-serve/Dockerfile @@ -39,6 +39,9 @@ RUN uv sync --frozen --no-dev $EXTRAS --no-editable --no-cache RUN groupadd --system --gid 10001 intern \ && useradd --system --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin intern USER intern +# The mount point for the Triton autotune cache volume, created HERE so a fresh named volume +# inherits intern:intern (a root-owned mount point would make every autotune write fail). +RUN mkdir -p /tmp/triton-cache ENV PATH=/app/.venv/bin:$PATH \ HF_HOME=/hf \ HF_HUB_OFFLINE=1 \ diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/README.md b/services/intern-decision-serve/acceptance/systemone-2026-09-30/README.md index 7e698bb..3ba5dea 100644 --- a/services/intern-decision-serve/acceptance/systemone-2026-09-30/README.md +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/README.md @@ -41,3 +41,17 @@ infra-ops audit PASS with two low findings, both fixed in 0.1.2: Re-run on 0.1.2 (jb2-* files, the 0.1.1 artifacts kept with their version suffix): all **202/231**, hard **83/111**, **0 changed rows** across 924. 124 tests green. Rollback: 0.1.1. + +## 0.1.3 — triton-cache volume + warm-up script (same day, Prime task) + +Image change only (Dockerfile creates /tmp/triton-cache owned 10001; compose mounts the named +volume there). Measured facts in the stack README "Cold-shape latency": bucket width 2,048 PROVEN, +16 buckets, cold full warm-up 109 s, warm re-run 17 s. Acceptance here: +- warmup-cold.txt / warmup-warm.txt: the two runs; every bucket in-bucket, cold 109.3 s vs 17.2 s. +- random-validation.txt: 12 random sizes 8.3k–31.8k after warm-up, worst wall 2.09 s — no cold call. +- Recreate survival: force-recreate, then a warmed 32k call answered in 2.11 s; volume holds + 3,200+ files owned by uid 10001 (new files newer than the image build = the volume is in use). +- Memory: warm 32k peak 15,218 MiB (budget 15,220); a COLD autotune touched 15,224 once, during + the pre-deploy 0.1.2-era measurement — see README caveat. gpu1-cold-run.csv, gpu1-32k-warm.csv. +- JevBench on 0.1.3: all 202/231, hard 83/111, 0 diffs / 924 (jb-results-0.1.3.jsonl). +- /decide positive control: answered (yes, 0.969). diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/gpu1-32k-warm.csv b/services/intern-decision-serve/acceptance/systemone-2026-09-30/gpu1-32k-warm.csv new file mode 100644 index 0000000..266e6f3 --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/gpu1-32k-warm.csv @@ -0,0 +1,600 @@ +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +15218 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8832 +8832 +8832 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8834 +8812 +9306 +9300 +8812 +9300 +9300 +8884 +9118 +9118 +9200 +9300 +8812 +9296 +9298 +8812 +8988 +9646 +9540 +9500 +8812 +9566 +9518 +9032 +9122 +9124 +9124 +9380 +9352 +9346 +8812 +9352 +8916 +8916 +8916 +8916 +9350 +8812 +9352 +8880 +8880 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +8944 +9294 +9346 +9346 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 +8812 diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/gpu1-cold-run.csv b/services/intern-decision-serve/acceptance/systemone-2026-09-30/gpu1-cold-run.csv new file mode 100644 index 0000000..6d62f08 --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/gpu1-cold-run.csv @@ -0,0 +1,1500 @@ +1790808847.496864309,8812 +1790808847.748366444,8812 +1790808848.001008930,8812 +1790808848.253405024,8812 +1790808848.506327434,8812 +1790808848.756020167,8812 +1790808849.009565555,8812 +1790808849.261989990,8812 +1790808849.514147193,8812 +1790808849.765599287,8812 +1790808850.016816068,8812 +1790808850.268367953,8812 +1790808850.519090068,8812 +1790808850.768495498,8812 +1790808851.019549558,8812 +1790808851.272639080,8812 +1790808851.524389747,8812 +1790808851.775866661,8812 +1790808852.026926470,8812 +1790808852.276718965,8812 +1790808852.525791121,8812 +1790808852.777170343,8812 +1790808853.028066710,8812 +1790808853.280088260,8812 +1790808853.531912108,8812 +1790808853.784707716,8812 +1790808854.036572454,8812 +1790808854.287971827,8812 +1790808854.540222900,8812 +1790808854.792034697,8812 +1790808855.045047719,8812 +1790808855.296486082,8812 +1790808855.548109928,8812 +1790808855.799553871,8812 +1790808856.051742543,8812 +1790808856.303282258,8812 +1790808856.553068752,8812 +1790808856.807798883,8812 +1790808857.063950420,8812 +1790808857.317492497,8812 +1790808857.571684992,8812 +1790808857.825308260,8812 +1790808858.077908297,8812 +1790808858.330696926,8812 +1790808858.584380055,8812 +1790808858.840362040,8812 +1790808859.095090711,8812 +1790808859.348001781,8812 +1790808859.601914532,8812 +1790808859.856320839,8812 +1790808860.111786189,8812 +1790808860.365177864,8812 +1790808860.619635542,8812 +1790808860.877498438,8812 +1790808861.131922145,8812 +1790808861.385832728,8812 +1790808861.640149484,8812 +1790808861.895477872,8812 +1790808862.151789561,8812 +1790808862.407685574,8812 +1790808862.664758591,8812 +1790808862.918502250,8812 +1790808863.173624276,8812 +1790808863.429073096,8812 +1790808863.685005770,8812 +1790808863.939652340,8812 +1790808864.195706526,8812 +1790808864.451574489,8812 +1790808864.707405692,8812 +1790808864.961541506,8812 +1790808865.215835012,8812 +1790808865.472281731,8812 +1790808865.727107473,8812 +1790808865.981432829,8812 +1790808866.238024432,8812 +1790808866.492788763,8812 +1790808866.746640573,8920 +1790808866.998827785,9556 +1790808867.250285489,9556 +1790808867.502000296,9556 +1790808867.756961079,9556 +1790808868.008363972,9556 +1790808868.261459324,9556 +1790808868.512289801,9556 +1790808868.765152630,9556 +1790808869.016677085,9556 +1790808869.268887357,9556 +1790808869.520346410,9556 +1790808869.772116057,9556 +1790808870.022074794,9556 +1790808870.277531503,9556 +1790808870.536945687,9556 +1790808870.789936398,9556 +1790808871.043198602,9556 +1790808871.294063440,9556 +1790808871.546761397,9556 +1790808871.803673033,9556 +1790808872.057587714,9556 +1790808872.307595012,9558 +1790808872.558959354,9558 +1790808872.815814939,9558 +1790808873.073512044,9558 +1790808873.322510069,9694 +1790808873.573645329,9210 +1790808873.824041711,9674 +1790808874.077523977,9674 +1790808874.329792610,9674 +1790808874.587056750,9674 +1790808874.837109027,9674 +1790808875.089079547,9674 +1790808875.338874422,9674 +1790808875.590962262,9674 +1790808875.841278823,9674 +1790808876.096322758,9674 +1790808876.355774302,9674 +1790808876.608135686,9674 +1790808876.861546052,9674 +1790808877.115555544,9674 +1790808877.368333343,9674 +1790808877.625715434,9674 +1790808877.883934825,9674 +1790808878.141826992,9674 +1790808878.396227609,9674 +1790808878.649060528,9674 +1790808878.905682900,9674 +1790808879.163250064,9674 +1790808879.417638961,9674 +1790808879.671991087,9674 +1790808879.934999932,9674 +1790808880.191713806,10014 +1790808880.447738271,8814 +1790808880.702013117,9660 +1790808880.953350069,9660 +1790808881.204114005,9660 +1790808881.456037224,9660 +1790808881.711057798,9660 +1790808881.961961185,9660 +1790808882.211833580,9660 +1790808882.463587648,9660 +1790808882.718735333,9660 +1790808882.976169715,9660 +1790808883.235197505,9660 +1790808883.492438514,9660 +1790808883.744157911,9660 +1790808883.996425103,9660 +1790808884.257110522,9660 +1790808884.515478784,9660 +1790808884.768315944,9660 +1790808885.024583712,9660 +1790808885.276788435,9660 +1790808885.536900306,9660 +1790808885.795933376,9660 +1790808886.051846231,9660 +1790808886.303776190,9660 +1790808886.562739049,9660 +1790808886.818781224,10142 +1790808887.072394593,10142 +1790808887.325434994,9802 +1790808887.580364297,9802 +1790808887.840560540,9802 +1790808888.097288404,9802 +1790808888.346093047,9802 +1790808888.601010520,9802 +1790808888.853198752,9802 +1790808889.101149736,9802 +1790808889.356260261,9802 +1790808889.613916875,9802 +1790808889.865739373,9802 +1790808890.116089694,9802 +1790808890.368549119,9802 +1790808890.621040734,9802 +1790808890.881022855,9802 +1790808891.138753690,9802 +1790808891.389846160,9802 +1790808891.640856758,9802 +1790808891.892716526,9802 +1790808892.149370189,9802 +1790808892.408235407,9802 +1790808892.666174584,9802 +1790808892.921336890,9802 +1790808893.170209544,9802 +1790808893.431288417,9802 +1790808893.691350179,10482 +1790808893.944157278,10482 +1790808894.197600934,8814 +1790808894.448761424,9950 +1790808894.702044689,9950 +1790808894.953691754,9950 +1790808895.209346286,9950 +1790808895.461573818,9950 +1790808895.711191771,9950 +1790808895.961964856,9950 +1790808896.215526354,9950 +1790808896.466613883,9950 +1790808896.720793768,9950 +1790808896.980326763,9950 +1790808897.238105169,9950 +1790808897.488619392,9950 +1790808897.745623409,9950 +1790808897.996605637,9950 +1790808898.245553562,9950 +1790808898.499363202,9950 +1790808898.747145834,9950 +1790808899.007627300,9950 +1790808899.265981132,9950 +1790808899.516558306,9950 +1790808899.768201092,9950 +1790808900.021219533,9950 +1790808900.277073817,9950 +1790808900.534044023,9950 +1790808900.784673558,10862 +1790808901.037366165,10862 +1790808901.290803451,10862 +1790808901.544537111,10096 +1790808901.804667293,10096 +1790808902.064415731,10096 +1790808902.325833428,10096 +1790808902.577764977,10096 +1790808902.828544233,10096 +1790808903.080672504,10096 +1790808903.331809474,10096 +1790808903.589188825,10096 +1790808903.842009554,10096 +1790808904.093114854,10096 +1790808904.344008571,10096 +1790808904.596159492,10096 +1790808904.856777500,10096 +1790808905.110996835,10096 +1790808905.363665153,10096 +1790808905.617363092,10096 +1790808905.869779556,10096 +1790808906.121837657,10096 +1790808906.381555634,10096 +1790808906.641619796,10096 +1790808906.894100262,10096 +1790808907.145839428,10096 +1790808907.407604499,10096 +1790808907.666143103,10096 +1790808907.918834741,11160 +1790808908.174935787,11160 +1790808908.429700819,11160 +1790808908.684037295,8814 +1790808908.937019106,10262 +1790808909.189741024,10262 +1790808909.442833836,10262 +1790808909.693094806,10262 +1790808909.944595210,10262 +1790808910.196093235,10262 +1790808910.448566750,10262 +1790808910.698339854,10262 +1790808910.950698268,10262 +1790808911.204649410,10262 +1790808911.465229969,10262 +1790808911.720855400,10262 +1790808911.972436985,10262 +1790808912.232544147,10262 +1790808912.482926768,10262 +1790808912.733511622,10262 +1790808912.986326511,10262 +1790808913.237579932,10264 +1790808913.489791934,10264 +1790808913.748886655,10264 +1790808914.006736341,10264 +1790808914.258967084,10264 +1790808914.510083883,10264 +1790808914.773066958,10264 +1790808915.032802636,10264 +1790808915.283060206,11768 +1790808915.535428050,11768 +1790808915.789264630,11768 +1790808916.044118093,11768 +1790808916.298224726,10152 +1790808916.550135315,10408 +1790808916.803939205,10408 +1790808917.055333599,10408 +1790808917.304462856,10408 +1790808917.565933373,10408 +1790808917.816771600,10408 +1790808918.067589196,10408 +1790808918.321362186,10408 +1790808918.579401595,10408 +1790808918.838678427,10408 +1790808919.093345787,10408 +1790808919.352000983,10408 +1790808919.601428013,10408 +1790808919.853047679,10408 +1790808920.107022681,10408 +1790808920.364963978,10408 +1790808920.627309856,10408 +1790808920.886426517,10408 +1790808921.144422905,10408 +1790808921.397414486,10408 +1790808921.648336133,10408 +1790808921.903778752,10408 +1790808922.162618540,10408 +1790808922.418010299,10408 +1790808922.668442190,10408 +1790808922.920045495,12532 +1790808923.172396999,12532 +1790808923.425638613,12532 +1790808923.677254819,12532 +1790808923.930855287,8816 +1790808924.181879716,10554 +1790808924.432501399,10554 +1790808924.684838373,10554 +1790808924.940562585,10554 +1790808925.190670393,10554 +1790808925.440183605,10554 +1790808925.690331494,10554 +1790808925.941311102,10554 +1790808926.193558104,10554 +1790808926.443473570,10554 +1790808926.699010980,10554 +1790808926.949714215,10554 +1790808927.200579882,10554 +1790808927.459851805,10554 +1790808927.720484733,10554 +1790808927.976437918,10554 +1790808928.228227975,10554 +1790808928.478340843,10554 +1790808928.734172077,10554 +1790808928.984203444,10554 +1790808929.244762492,10554 +1790808929.504349957,10554 +1790808929.767439904,10554 +1790808930.020277433,10554 +1790808930.272558806,10554 +1790808930.530104319,10554 +1790808930.797316062,12914 +1790808931.050912840,12914 +1790808931.304044292,12914 +1790808931.559279059,12914 +1790808931.809793812,8816 +1790808932.060339305,10700 +1790808932.322362409,10700 +1790808932.574136866,10700 +1790808932.832324286,10700 +1790808933.083169083,10700 +1790808933.334024070,10700 +1790808933.586195921,10700 +1790808933.834913514,10700 +1790808934.087922415,10700 +1790808934.340242999,10700 +1790808934.592510221,10700 +1790808934.843657241,10700 +1790808935.093439296,10700 +1790808935.344271652,10700 +1790808935.605680539,10700 +1790808935.866149535,10700 +1790808936.121444292,10700 +1790808936.371506091,10700 +1790808936.632154478,10700 +1790808936.882999885,10700 +1790808937.132837161,10700 +1790808937.387990176,10700 +1790808937.643098071,10700 +1790808937.899240218,10700 +1790808938.156280285,10700 +1790808938.407905931,10700 +1790808938.661648450,13296 +1790808938.914917675,13296 +1790808939.167876135,13296 +1790808939.420099278,13296 +1790808939.676030652,13296 +1790808939.930798483,8816 +1790808940.182736172,10846 +1790808940.432795800,10846 +1790808940.683290383,10846 +1790808940.938944674,10846 +1790808941.189088953,10846 +1790808941.439565235,10846 +1790808941.689428931,10846 +1790808941.939666131,10846 +1790808942.193039436,10846 +1790808942.444718482,10846 +1790808942.696993515,10846 +1790808942.949237167,10846 +1790808943.200476718,10846 +1790808943.454865945,10846 +1790808943.713455610,10846 +1790808943.971400268,10846 +1790808944.227071809,10846 +1790808944.478812976,10846 +1790808944.738954438,10846 +1790808944.990794206,10846 +1790808945.240633862,10846 +1790808945.496431885,10846 +1790808945.759165917,10846 +1790808946.015040251,10846 +1790808946.267458635,10846 +1790808946.520617478,10846 +1790808946.773074403,13678 +1790808947.026871683,13678 +1790808947.279733223,13678 +1790808947.533481022,13678 +1790808947.789162884,13678 +1790808948.041010222,13678 +1790808948.293644339,10736 +1790808948.544596017,10992 +1790808948.802097979,10992 +1790808949.052964847,10992 +1790808949.305107168,10992 +1790808949.566118790,10992 +1790808949.815898244,10992 +1790808950.065856541,10992 +1790808950.318868493,10992 +1790808950.579538141,10992 +1790808950.840069678,10992 +1790808951.101301003,10992 +1790808951.361357964,10992 +1790808951.612665496,10992 +1790808951.864773347,10992 +1790808952.118235064,10992 +1790808952.376583225,10992 +1790808952.635442453,10992 +1790808952.890946793,10992 +1790808953.148782180,10992 +1790808953.406533555,10992 +1790808953.657034697,10992 +1790808953.908918026,10992 +1790808954.160710243,10992 +1790808954.418826633,10992 +1790808954.677020443,10992 +1790808954.932481152,14060 +1790808955.186815478,14060 +1790808955.438006199,14060 +1790808955.689948688,14060 +1790808955.944261944,14060 +1790808956.198162976,14060 +1790808956.451244088,14060 +1790808956.705623965,10886 +1790808956.960379136,11142 +1790808957.217041028,11142 +1790808957.469119989,11142 +1790808957.721517964,11142 +1790808957.979805995,11142 +1790808958.232816176,11142 +1790808958.484136438,11142 +1790808958.737629415,11142 +1790808958.998266983,11142 +1790808959.257712478,11142 +1790808959.518150274,11142 +1790808959.777928222,11142 +1790808960.028779748,11142 +1790808960.284076446,11142 +1790808960.535853883,11142 +1790808960.789904516,11142 +1790808961.043775237,11142 +1790808961.294331251,11142 +1790808961.547970059,11142 +1790808961.815483085,11142 +1790808962.072047257,11142 +1790808962.323263138,11142 +1790808962.574275456,11142 +1790808962.828007836,11142 +1790808963.092215915,11144 +1790808963.354573532,11144 +1790808963.611594110,14448 +1790808963.868551706,14448 +1790808964.120836539,14448 +1790808964.375026024,14448 +1790808964.628456740,14448 +1790808964.884000330,14448 +1790808965.138254285,8818 +1790808965.390678910,11290 +1790808965.648612157,11290 +1790808965.904858755,11290 +1790808966.167357944,11290 +1790808966.417977288,11290 +1790808966.668559512,11290 +1790808966.920393250,11290 +1790808967.171518760,11290 +1790808967.428412205,11290 +1790808967.686729967,11290 +1790808967.938616165,11290 +1790808968.188404160,11290 +1790808968.445565298,11290 +1790808968.697392886,11290 +1790808968.946653515,11290 +1790808969.204477161,11290 +1790808969.465961319,11290 +1790808969.726064581,11290 +1790808969.984416763,11290 +1790808970.244708627,11290 +1790808970.495673605,11290 +1790808970.747924668,11290 +1790808971.000061329,11290 +1790808971.252931459,11290 +1790808971.509828434,11290 +1790808971.767360987,11290 +1790808972.019196605,14830 +1790808972.271578939,14830 +1790808972.522519117,14830 +1790808972.775852642,14830 +1790808973.029138846,14830 +1790808973.283568293,14830 +1790808973.536204341,14830 +1790808973.789521185,14830 +1790808974.042311024,11192 +1790808974.293807948,11448 +1790808974.544333471,11448 +1790808974.794080295,11448 +1790808975.044674609,11448 +1790808975.295913250,11448 +1790808975.548133622,11448 +1790808975.799022589,11448 +1790808976.048557941,11448 +1790808976.300266527,11448 +1790808976.553964116,11448 +1790808976.808290333,11448 +1790808977.060930870,11448 +1790808977.312410213,11448 +1790808977.579225782,11448 +1790808977.838913700,11448 +1790808978.089513294,11448 +1790808978.340607683,11448 +1790808978.592001816,11448 +1790808978.848658039,11448 +1790808979.100956492,11448 +1790808979.357733916,11448 +1790808979.616869277,11448 +1790808979.877599356,11448 +1790808980.132040194,11448 +1790808980.384571999,11448 +1790808980.638635413,11448 +1790808980.897305429,15224 +1790808981.149397269,15224 +1790808981.401690222,15224 +1790808981.653478340,15224 +1790808981.905886284,15224 +1790808982.159250469,15224 +1790808982.413279552,15224 +1790808982.666525596,15224 +1790808982.919126183,8818 +1790808983.168947188,8818 +1790808983.420493483,8818 +1790808983.671575762,8818 +1790808983.924117188,8818 +1790808984.176208338,8818 +1790808984.426321607,8818 +1790808984.679464139,8818 +1790808984.929668169,8818 +1790808985.180158872,8818 +1790808985.430856206,8818 +1790808985.679801431,8818 +1790808985.931942572,8818 +1790808986.184001353,8818 +1790808986.436046153,8818 +1790808986.685295752,8818 +1790808986.937455334,8818 +1790808987.187837115,8818 +1790808987.439606612,8818 +1790808987.690930954,8818 +1790808987.942615920,8818 +1790808988.192372425,8818 +1790808988.440675742,8818 +1790808988.691600999,8818 +1790808988.941734778,8818 +1790808989.190456840,8818 +1790808989.438979081,8818 +1790808989.688881626,8818 +1790808989.940053897,8818 +1790808990.189966853,8818 +1790808990.438095939,8818 +1790808990.685704038,8818 +1790808990.933472110,8818 +1790808991.182015760,8818 +1790808991.428991003,8818 +1790808991.678332522,8818 +1790808991.926735411,8818 +1790808992.175515184,8818 +1790808992.424490319,8818 +1790808992.672415503,8818 +1790808992.922040646,8818 +1790808993.171295314,8818 +1790808993.419317969,8818 +1790808993.667465925,8818 +1790808993.916268088,8818 +1790808994.164434644,8818 +1790808994.413000795,8818 +1790808994.662572357,8818 +1790808994.911381160,8818 +1790808995.159517006,8818 +1790808995.410259631,8818 +1790808995.660152567,8818 +1790808995.909640098,8818 +1790808996.157990616,8818 +1790808996.406173533,8818 +1790808996.654127006,8818 +1790808996.902524985,8818 +1790808997.151936176,8818 +1790808997.401559708,8818 +1790808997.650933838,8818 +1790808997.898293505,8818 +1790808998.145882635,8818 +1790808998.395690719,8818 +1790808998.642190797,8818 +1790808998.889936998,8818 +1790808999.137962502,8818 +1790808999.384643051,8818 +1790808999.635730071,8818 +1790808999.887564959,8818 +1790809000.139162274,8818 +1790809000.389522255,8818 +1790809000.637606000,8818 +1790809000.889656371,8818 +1790809001.140361814,8818 +1790809001.391897679,8818 +1790809001.643695056,8818 +1790809001.895816217,8818 +1790809002.147797127,8818 +1790809002.399656164,8818 +1790809002.650403339,8818 +1790809002.902153407,8818 +1790809003.152209384,8818 +1790809003.403398825,8818 +1790809003.654897199,8818 +1790809003.905635444,8818 +1790809004.156105065,8818 +1790809004.407567790,8818 +1790809004.658750041,8818 +1790809004.908863819,8818 +1790809005.159290751,8818 +1790809005.409512891,8818 +1790809005.657922260,8818 +1790809005.909088450,8818 +1790809006.160504733,8818 +1790809006.412364872,8818 +1790809006.664720236,8818 +1790809006.915861284,8818 +1790809007.169209079,8818 +1790809007.419457639,8926 +1790809007.669639129,9958 +1790809007.923146535,9734 +1790809008.174917783,10018 +1790809008.427583810,10418 +1790809008.681357040,10418 +1790809008.933810715,10810 +1790809009.187446013,10810 +1790809009.443153465,11210 +1790809009.696835654,11210 +1790809009.952244583,11210 +1790809010.207916634,11608 +1790809010.461575823,11608 +1790809010.716417245,11608 +1790809010.969125783,12026 +1790809011.223333368,12026 +1790809011.475295687,12026 +1790809011.732350355,12026 +1790809011.986752672,12422 +1790809012.240028296,12422 +1790809012.493042217,12422 +1790809012.745281580,12422 +1790809012.998035828,8818 +1790809013.249007156,12820 +1790809013.500931405,12820 +1790809013.756211732,12820 +1790809014.011175705,12820 +1790809014.264366759,8818 +1790809014.516999766,13218 +1790809014.772373864,13218 +1790809015.027806513,13218 +1790809015.279575120,13218 +1790809015.530703610,13218 +1790809015.783295416,13616 +1790809016.036374378,13616 +1790809016.289365460,13616 +1790809016.543153360,13616 +1790809016.796012369,13616 +1790809017.052702482,13616 +1790809017.305971566,14014 +1790809017.560601986,14014 +1790809017.812467274,14014 +1790809018.067969074,14014 +1790809018.321868465,14014 +1790809018.574871607,14014 +1790809018.828259362,14014 +1790809019.082022672,14416 +1790809019.334712600,14416 +1790809019.586393816,14416 +1790809019.838656269,14416 +1790809020.091051513,14416 +1790809020.343264145,14416 +1790809020.600274992,14416 +1790809020.856367229,14814 +1790809021.115536910,14814 +1790809021.368461190,14814 +1790809021.624105761,14814 +1790809021.877512507,14814 +1790809022.131532250,14814 +1790809022.385352000,14814 +1790809022.640276893,14814 +1790809022.893801910,15224 +1790809023.148564582,15224 +1790809023.404208513,15224 +1790809023.657765280,15224 +1790809023.912030256,15224 +1790809024.165831496,15224 +1790809024.418821087,15224 +1790809024.672765939,15224 +1790809024.925500917,8818 +1790809025.176827779,8818 +1790809025.428544586,8818 +1790809025.679647306,8818 +1790809025.929832705,8818 +1790809026.179001143,8818 +1790809026.428640945,8818 +1790809026.680070479,8818 +1790809026.931755345,8818 +1790809027.182658503,8818 +1790809027.432982303,8818 +1790809027.683477776,8818 +1790809027.933332371,8818 +1790809028.184432881,8818 +1790809028.435598761,8818 +1790809028.687178576,8818 +1790809028.935299731,8818 +1790809029.185505861,8818 +1790809029.436629421,8818 +1790809029.686959092,8818 +1790809029.937766787,8818 +1790809030.187878066,8818 +1790809030.435768818,8818 +1790809030.687445655,8818 +1790809030.939155781,8818 +1790809031.187248206,8818 +1790809031.435477193,8818 +1790809031.684062285,8818 +1790809031.933251031,8818 +1790809032.183935316,8818 +1790809032.432380564,8818 +1790809032.680770273,8818 +1790809032.929613327,8818 +1790809033.178155867,8818 +1790809033.427160383,8818 +1790809033.675527882,8818 +1790809033.923776899,8818 +1790809034.172564472,8818 +1790809034.421198284,8818 +1790809034.669977087,8818 +1790809034.918494897,8818 +1790809035.166052757,8818 +1790809035.413833158,8818 +1790809035.663014505,8818 +1790809035.913216565,8818 +1790809036.161527343,8818 +1790809036.411303827,8818 +1790809036.658003837,8818 +1790809036.906470266,8818 +1790809037.154672203,8818 +1790809037.402975610,8818 +1790809037.651641042,8818 +1790809037.901828271,8818 +1790809038.149826625,8818 +1790809038.396671097,8818 +1790809038.645838434,8818 +1790809038.893478294,8818 +1790809039.142136986,8818 +1790809039.391645867,8818 +1790809039.641750575,8818 +1790809039.890943153,8818 +1790809040.139102019,8818 +1790809040.386889471,8818 +1790809040.635005506,8818 +1790809040.883545177,8818 +1790809041.131668663,8818 +1790809041.382000833,8818 +1790809041.633448697,8818 +1790809041.884212322,8818 +1790809042.136401104,8818 +1790809042.386966618,8818 +1790809042.637741654,8818 +1790809042.889923086,8818 +1790809043.140674421,8818 +1790809043.392889683,8818 +1790809043.642398814,8818 +1790809043.892043488,8818 +1790809044.142235607,8818 +1790809044.391979011,8818 +1790809044.644444976,8818 +1790809044.896123352,8818 +1790809045.146004038,8818 +1790809045.398056788,8818 +1790809045.648208307,8818 +1790809045.900930275,8818 +1790809046.153239918,8818 +1790809046.405443590,8818 +1790809046.656658611,8818 +1790809046.910191688,8818 +1790809047.161209186,8818 +1790809047.412830882,8818 +1790809047.664361507,8818 +1790809047.916307026,8818 +1790809048.177953305,10446 +1790809048.430045776,10998 +1790809048.684445463,10998 +1790809048.935842836,8820 +1790809049.189119291,11608 +1790809049.444161185,11608 +1790809049.699493352,11608 +1790809049.955341526,11988 +1790809050.212899409,11988 +1790809050.465889980,11988 +1790809050.717932920,11988 +1790809050.972393378,12256 +1790809051.228407254,12256 +1790809051.483770222,12256 +1790809051.740944211,12256 +1790809051.995628841,13066 +1790809052.248872755,13066 +1790809052.502634485,13066 +1790809052.755202801,13066 +1790809053.007622585,13066 +1790809053.261200583,13124 +1790809053.516315398,13124 +1790809053.770127199,13124 +1790809054.023448324,13124 +1790809054.277900741,13124 +1790809054.531508269,8818 +1790809054.783781032,13352 +1790809055.039883869,13352 +1790809055.293362655,13352 +1790809055.548109856,13352 +1790809055.803087610,13352 +1790809056.054502483,8818 +1790809056.309490567,13598 +1790809056.559574095,13598 +1790809056.813028031,13598 +1790809057.064082910,13598 +1790809057.317761189,13598 +1790809057.571585690,8818 +1790809057.824339778,13616 +1790809058.080006539,13616 +1790809058.333585507,13616 +1790809058.586264685,13616 +1790809058.836593296,13616 +1790809059.089617327,8818 +1790809059.340021439,13696 +1790809059.596190586,13696 +1790809059.850715424,13696 +1790809060.101486180,13696 +1790809060.354748934,13696 +1790809060.615978409,13696 +1790809060.867320512,15046 +1790809061.120309473,15046 +1790809061.376257267,15046 +1790809061.632388054,15046 +1790809061.888070995,15046 +1790809062.139629520,15046 +1790809062.391505689,15046 +1790809062.643880423,15046 +1790809062.898339801,8818 +1790809063.148626841,8818 +1790809063.399246925,8818 +1790809063.647893286,8818 +1790809063.900197139,8818 +1790809064.151886406,8818 +1790809064.403771895,8818 +1790809064.653880983,8818 +1790809064.904383205,8818 +1790809065.153955967,8818 +1790809065.404181678,8818 +1790809065.655869714,8818 +1790809065.908230217,8818 +1790809066.157902460,8818 +1790809066.408931209,8818 +1790809066.661199712,8818 +1790809066.912593705,8818 +1790809067.161025894,8818 +1790809067.409147230,8818 +1790809067.658590110,8818 +1790809067.909332905,8818 +1790809068.159871129,8818 +1790809068.411822728,8818 +1790809068.662852657,8818 +1790809068.914906537,8818 +1790809069.164673241,8818 +1790809069.415861722,8818 +1790809069.668431418,8818 +1790809069.918949551,8818 +1790809070.167763814,8818 +1790809070.415543986,8818 +1790809070.661153643,8818 +1790809070.911679876,8818 +1790809071.160666171,8818 +1790809071.409484465,8818 +1790809071.657392228,8818 +1790809071.905230020,0 +1790809072.161624390,0 +1790809072.409822246,0 +1790809072.656068031,0 +1790809072.904899775,0 +1790809073.151935378,0 +1790809073.400514828,0 +1790809073.651777840,0 +1790809073.900820606,0 +1790809074.149437567,0 +1790809074.400082922,0 +1790809074.649478001,0 +1790809074.901537402,0 +1790809075.151504498,0 +1790809075.401400244,0 +1790809075.650220978,0 +1790809075.901706442,0 +1790809076.151748019,0 +1790809076.404451807,0 +1790809076.656803581,0 +1790809076.907295513,0 +1790809077.156905346,0 +1790809077.408480251,0 +1790809077.660617993,0 +1790809077.914120759,0 +1790809078.164012165,0 +1790809078.417800595,0 +1790809078.667354277,0 +1790809078.919144534,0 +1790809079.168226581,0 +1790809079.419735495,0 +1790809079.671584943,0 +1790809079.921796753,0 +1790809080.171881271,0 +1790809080.424644729,0 +1790809080.676079933,0 +1790809080.928663889,0 +1790809081.180816241,0 +1790809081.431247842,0 +1790809081.682095209,0 +1790809081.932448690,0 +1790809082.184482360,0 +1790809082.435974424,0 +1790809082.687463628,0 +1790809082.939555139,0 +1790809083.192333088,0 +1790809083.445708303,0 +1790809083.694984392,0 +1790809083.943624213,0 +1790809084.194160616,0 +1790809084.444004412,0 +1790809084.695226742,0 +1790809084.944871175,0 +1790809085.193178803,0 +1790809085.441325389,0 +1790809085.690078662,0 +1790809085.938886745,0 +1790809086.186931380,0 +1790809086.434024554,0 +1790809086.680917635,0 +1790809086.932613672,0 +1790809087.183482759,0 +1790809087.431871537,0 +1790809087.683588964,0 +1790809087.932037073,0 +1790809088.182969861,0 +1790809088.432922927,0 +1790809088.684756095,0 +1790809088.936499272,0 +1790809089.188567892,0 +1790809089.441075518,0 +1790809089.691583350,0 +1790809089.944554232,0 +1790809090.197091898,0 +1790809090.447921364,0 +1790809090.700018345,0 +1790809090.952079035,0 +1790809091.204438219,0 +1790809091.454952972,0 +1790809091.706010061,0 +1790809091.955281260,0 +1790809092.205299517,0 +1790809092.456594688,0 +1790809092.706104190,0 +1790809092.956891026,0 +1790809093.206946044,0 +1790809093.457057402,0 +1790809093.707252681,0 +1790809093.958137848,0 +1790809094.207741460,0 +1790809094.458231463,0 +1790809094.708805387,0 +1790809094.957683731,0 +1790809095.207378664,0 +1790809095.458041099,0 +1790809095.708245518,0 +1790809095.960836585,0 +1790809096.211660121,0 +1790809096.461392505,0 +1790809096.711872017,0 +1790809096.963287840,0 +1790809097.213863024,0 +1790809097.465739292,0 +1790809097.714889680,0 +1790809097.965522064,0 +1790809098.216354640,0 +1790809098.465722070,0 +1790809098.714262801,0 +1790809098.964278488,0 +1790809099.215478718,0 +1790809099.464666386,0 +1790809099.714655023,0 +1790809099.965673052,0 +1790809100.215261444,0 +1790809100.464248380,0 +1790809100.714906564,0 +1790809100.964602127,0 +1790809101.215625966,0 +1790809101.464679952,0 +1790809101.715823843,0 +1790809101.965966191,0 +1790809102.217558086,0 +1790809102.466228818,0 +1790809102.719594253,0 +1790809102.974118351,0 +1790809103.226089581,0 +1790809103.477850818,0 +1790809103.730445515,0 +1790809103.983264974,0 +1790809104.247982028,0 +1790809104.504096075,0 +1790809104.757821554,0 +1790809105.012996690,0 +1790809105.266937852,0 +1790809105.520618481,0 +1790809105.775362332,0 +1790809106.032865195,0 +1790809106.283188785,0 +1790809106.532648576,0 +1790809106.783480842,0 +1790809107.033589500,0 +1790809107.284487028,0 +1790809107.535667409,0 +1790809107.788138674,0 +1790809108.039426095,0 +1790809108.291510296,0 +1790809108.543179202,0 +1790809108.793460482,0 +1790809109.043024084,0 +1790809109.293092482,0 +1790809109.543461133,0 +1790809109.793867905,0 +1790809110.044069934,0 +1790809110.295421916,0 +1790809110.545881559,0 +1790809110.796244210,0 +1790809111.048135928,0 +1790809111.299435950,0 +1790809111.549292195,0 +1790809111.800515016,0 +1790809112.052032040,0 +1790809112.302035547,0 +1790809112.549472515,0 +1790809112.797788813,0 +1790809113.046916810,0 +1790809113.295737313,0 +1790809113.548094187,0 +1790809113.800462881,0 +1790809114.052151058,0 +1790809114.304071396,0 +1790809114.555718573,0 +1790809114.807340488,0 +1790809115.059442779,0 +1790809115.305438770,0 +1790809115.552957899,0 +1790809115.801639441,0 +1790809116.049488653,0 +1790809116.297967433,0 +1790809116.545952197,0 +1790809116.795268667,0 +1790809117.042843096,0 +1790809117.292707633,0 +1790809117.544102576,0 +1790809117.799793357,0 +1790809118.048421388,0 +1790809118.297763138,0 +1790809118.550202993,0 +1790809118.801584566,0 +1790809119.054963741,0 +1790809119.303753324,0 +1790809119.556804656,0 +1790809119.807059825,0 +1790809120.055085750,0 +1790809120.303425848,0 +1790809120.553351914,0 +1790809120.801880544,0 +1790809121.049363973,0 +1790809121.300591834,0 +1790809121.549550799,0 +1790809121.800557697,0 +1790809122.049416251,0 +1790809122.297844650,0 +1790809122.545296417,0 +1790809122.796890873,0 +1790809123.046938110,0 +1790809123.296215799,0 +1790809123.546613551,0 +1790809123.795548565,0 +1790809124.047767178,0 +1790809124.303166386,0 +1790809124.555896585,0 +1790809124.807912494,0 +1790809125.059566280,0 +1790809125.313542242,0 +1790809125.564435299,0 +1790809125.817435901,0 +1790809126.067543939,0 +1790809126.319626620,0 +1790809126.570996362,0 +1790809126.822039421,0 +1790809127.073428234,0 +1790809127.325386754,0 +1790809127.576884238,0 +1790809127.828613984,0 +1790809128.080582604,0 +1790809128.333037489,0 +1790809128.584847126,0 +1790809128.839273124,0 +1790809129.090883229,0 +1790809129.341923508,0 +1790809129.595017369,0 +1790809129.845964757,0 +1790809130.098841607,0 +1790809130.351268291,0 +1790809130.603646395,0 +1790809130.857809080,0 +1790809131.110166313,0 +1790809131.362067321,0 +1790809131.614645198,0 +1790809131.869792793,0 +1790809132.124441673,0 +1790809132.380880144,0 +1790809132.633808454,0 +1790809132.888319602,0 +1790809133.143635890,0 +1790809133.398804806,0 +1790809133.652610556,0 +1790809133.907209256,0 +1790809134.162815607,0 +1790809134.418007982,0 +1790809134.671635640,0 +1790809134.924827354,0 +1790809135.180365203,0 +1790809135.437671033,0 +1790809135.692221632,0 +1790809135.945805870,0 +1790809136.199917784,0 +1790809136.455901909,0 +1790809136.710604170,0 +1790809136.964319239,0 +1790809137.217293560,0 +1790809137.471807388,0 +1790809137.724836720,0 +1790809137.977845271,0 +1790809138.230489778,0 +1790809138.483805203,0 +1790809138.738872037,0 +1790809138.993075392,0 +1790809139.244113911,0 +1790809139.499861004,0 +1790809139.755233312,0 +1790809140.010045513,0 +1790809140.264520991,0 +1790809140.519070590,0 +1790809140.772244663,0 +1790809141.026397468,0 +1790809141.277936732,0 +1790809141.531004094,0 +1790809141.782113424,0 +1790809142.035967264,0 +1790809142.295930965,0 +1790809142.553090504,0 +1790809142.812267415,0 +1790809143.070237243,0 +1790809143.325365908,0 +1790809143.583613749,0 +1790809143.842993093,0 +1790809144.099366033,0 +1790809144.351630376,0 +1790809144.607668571,0 +1790809144.861854716,0 +1790809145.117052342,0 +1790809145.370588559,0 +1790809145.624361779,0 +1790809145.877500232,0 +1790809146.129766845,0 +1790809146.382956258,0 +1790809146.638249225,0 +1790809146.891052484,0 +1790809147.143260696,0 +1790809147.397502842,0 +1790809147.654632000,0 +1790809147.907263527,0 +1790809148.170427794,0 +1790809148.429920169,0 +1790809148.682577986,0 +1790809148.941623776,0 +1790809149.194265583,0 +1790809149.451069508,0 +1790809149.704443023,0 +1790809149.957322193,0 +1790809150.214092017,0 +1790809150.464286046,0 +1790809150.715573888,0 +1790809150.965284971,0 +1790809151.213928323,0 +1790809151.464500867,0 +1790809151.714230200,0 +1790809151.964309158,0 +1790809152.215651740,0 +1790809152.464158781,0 +1790809152.716650286,0 +1790809152.969030350,0 +1790809153.219160419,0 +1790809153.466949030,0 +1790809153.719583117,0 +1790809153.967911875,0 +1790809154.220087567,0 +1790809154.471185057,0 +1790809154.721211154,0 +1790809154.973403016,0 +1790809155.223776997,0 +1790809155.474920857,0 +1790809155.725719223,0 +1790809155.977762953,0 +1790809156.227376786,0 +1790809156.475818355,0 +1790809156.726961355,0 +1790809156.976374236,0 +1790809157.227186732,0 +1790809157.477168238,0 +1790809157.728244568,0 +1790809157.979296727,0 +1790809158.229413455,0 +1790809158.480387813,0 +1790809158.731843917,0 +1790809158.981489829,0 +1790809159.232108243,0 +1790809159.481951199,0 +1790809159.733580525,0 +1790809159.985444093,0 +1790809160.236465421,0 +1790809160.485991423,0 +1790809160.738935503,0 +1790809160.993116518,0 +1790809161.245683084,0 +1790809161.498890728,0 +1790809161.751055249,0 +1790809162.002879297,0 +1790809162.255811898,0 +1790809162.510596229,0 +1790809162.764286198,0 +1790809163.017953157,0 +1790809163.272151791,0 +1790809163.525307224,0 +1790809163.778850631,0 +1790809164.031823452,0 +1790809164.290610019,0 +1790809164.544381108,0 +1790809164.797833165,0 +1790809165.051780427,0 +1790809165.308775663,0 +1790809165.564351314,0 +1790809165.815776837,0 +1790809166.067863768,0 +1790809166.322837232,0 +1790809166.576395229,0 +1790809166.831548525,0 +1790809167.086003002,0 +1790809167.340490401,0 +1790809167.594753306,0 +1790809167.854228951,0 +1790809168.107142101,0 +1790809168.358892518,0 +1790809168.612007191,0 +1790809168.864462355,0 +1790809169.118062393,0 +1790809169.369902602,0 +1790809169.624850105,0 +1790809169.879165081,0 +1790809170.132541016,0 +1790809170.385072643,0 +1790809170.638542379,0 +1790809170.893273510,0 +1790809171.148874150,0 +1790809171.404847735,0 +1790809171.658207991,0 +1790809171.909951228,0 +1790809172.162955389,0 +1790809172.416602558,0 +1790809172.671732003,0 +1790809172.926366883,0 +1790809173.180136992,0 +1790809173.434822253,0 +1790809173.684290764,0 +1790809173.936787919,0 +1790809174.188306123,0 +1790809174.439773177,0 +1790809174.691076469,0 +1790809174.940063254,0 +1790809175.192137424,0 +1790809175.444148285,0 +1790809175.693912589,0 +1790809175.942916524,0 +1790809176.192419775,0 +1790809176.439460189,0 +1790809176.689532297,0 +1790809176.940200911,0 +1790809177.188703251,0 +1790809177.438965001,0 +1790809177.689474904,0 +1790809177.939425051,0 +1790809178.189900133,0 +1790809178.440488576,0 +1790809178.691668717,0 +1790809178.940469140,0 +1790809179.191015734,0 +1790809179.440664156,0 +1790809179.690983387,0 +1790809179.940475659,0 +1790809180.191166183,0 +1790809180.441568405,0 +1790809180.692982628,0 +1790809180.944699924,0 +1790809181.194555610,0 +1790809181.443639336,0 +1790809181.694253400,0 +1790809181.943400497,0 +1790809182.192361673,0 +1790809182.444940139,0 +1790809182.696136340,0 +1790809182.947115078,0 +1790809183.195280764,0 +1790809183.441227105,0 +1790809183.687481849,0 +1790809183.937233223,0 +1790809184.185133877,0 +1790809184.432109019,0 +1790809184.677795977,0 +1790809184.926034654,0 +1790809185.172068045,0 +1790809185.419109059,0 +1790809185.669294288,0 +1790809185.916218740,0 +1790809186.163631208,0 +1790809186.413339071,0 +1790809186.659882419,0 +1790809186.908597961,0 +1790809187.157546976,0 +1790809187.405665982,0 +1790809187.653003558,0 +1790809187.902496530,0 +1790809188.149640234,0 +1790809188.397522467,0 +1790809188.644600801,0 +1790809188.891252250,0 +1790809189.139440776,0 +1790809189.386287697,0 +1790809189.633684434,0 +1790809189.881292274,0 +1790809190.129529171,0 +1790809190.377934370,0 +1790809190.632212925,0 +1790809190.883126993,0 +1790809191.132032597,0 +1790809191.383423060,0 +1790809191.632592997,0 +1790809191.882757236,0 +1790809192.135525755,0 +1790809192.387429514,0 +1790809192.640652107,0 +1790809192.893264944,0 +1790809193.144529846,0 +1790809193.396451925,0 +1790809193.647617105,0 +1790809193.901252633,0 +1790809194.153646857,0 +1790809194.404153010,0 +1790809194.656639235,0 +1790809194.909727107,0 +1790809195.160376932,0 +1790809195.412022537,0 +1790809195.661187554,0 +1790809195.910004897,0 +1790809196.158412746,0 +1790809196.409760149,0 +1790809196.660631866,0 +1790809196.912693906,0 +1790809197.164471903,0 +1790809197.416917528,0 +1790809197.668827557,0 +1790809197.919983547,0 +1790809198.171188568,0 +1790809198.424332411,0 +1790809198.676210469,0 +1790809198.927800274,0 +1790809199.179750233,0 +1790809199.431296828,0 +1790809199.680504866,0 +1790809199.932160652,0 +1790809200.181890746,0 +1790809200.432107575,0 +1790809200.685322079,0 +1790809200.936292687,0 +1790809201.187362326,0 +1790809201.438783989,0 +1790809201.688914208,0 +1790809201.941410753,0 +1790809202.193372152,0 +1790809202.445103469,0 +1790809202.697389472,0 +1790809202.948388320,0 +1790809203.197591468,0 +1790809203.447423924,0 +1790809203.698949538,0 +1790809203.950949768,0 +1790809204.200825663,0 +1790809204.452335857,0 +1790809204.705416059,0 +1790809204.955681509,0 +1790809205.207725930,0 +1790809205.459670869,0 +1790809205.708635544,0 +1790809205.960249499,0 +1790809206.212917527,0 +1790809206.465210270,0 +1790809206.716065176,0 +1790809206.968423951,0 +1790809207.220642543,0 +1790809207.473825476,0 +1790809207.724762254,0 +1790809207.976340629,0 +1790809208.228324978,0 +1790809208.480036874,0 +1790809208.732507470,0 +1790809208.982493707,0 +1790809209.232284301,0 +1790809209.484114329,0 +1790809209.734181067,0 +1790809209.985538179,0 +1790809210.236499697,0 +1790809210.488993642,0 +1790809210.740414306,0 +1790809210.991621237,0 +1790809211.243614936,0 +1790809211.495315143,0 +1790809211.746395322,0 +1790809211.999517224,0 +1790809212.252474945,0 +1790809212.505410045,0 +1790809212.757519756,0 +1790809213.007249941,0 +1790809213.259000577,0 +1790809213.508081874,0 +1790809213.760451098,0 +1790809214.013610631,0 +1790809214.266313008,0 +1790809214.519266099,0 +1790809214.771452081,0 +1790809215.026918340,0 +1790809215.277721166,0 +1790809215.528459512,0 +1790809215.778997305,0 +1790809216.032532312,0 +1790809216.285673975,0 +1790809216.539249513,0 +1790809216.789716605,0 +1790809217.040476050,0 +1790809217.294150599,0 +1790809217.545800385,0 +1790809217.796511941,0 +1790809218.048385659,0 +1790809218.300970396,0 +1790809218.554131259,0 +1790809218.804573100,0 +1790809219.056506789,0 +1790809219.307174974,0 +1790809219.559115773,0 +1790809219.810307304,0 +1790809220.060489853,0 +1790809220.311808355,0 +1790809220.563638443,0 +1790809220.814297307,0 +1790809221.064802540,0 +1790809221.316616548,0 +1790809221.569843922,0 +1790809221.821222375,0 +1790809222.071519445,0 +1790809222.321758074,0 +1790809222.573141327,0 +1790809222.825770454,0 +1790809223.078228999,0 +1790809223.329223238,0 +1790809223.580631151,0 +1790809223.832533139,0 +1790809224.085762043,0 +1790809224.338237198,0 +1790809224.587552447,0 +1790809224.840281915,0 +1790809225.091213473,0 +1790809225.341979149,0 +1790809225.592763035,0 +1790809225.845249630,0 diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/jb-results-0.1.3.jsonl b/services/intern-decision-serve/acceptance/systemone-2026-09-30/jb-results-0.1.3.jsonl new file mode 100644 index 0000000..7bd2e1c --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/jb-results-0.1.3.jsonl @@ -0,0 +1,231 @@ +{"task_id": "easy-intent-00", "family": "intent", "split": "public", "group": null, "ts": 1790809199.5510135, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "track_order", "ordinal_ev": null, "probs": {"billing_question": 0.0037157811604703298, "cancel_order": 0.003078293568895903, "change_address": 0.001361767731259195, "report_damage": 0.0032776061174943907, "track_order": 0.9885665514218801}, "probs_as_returned": {"billing_question": 0.0037157811604703298, "cancel_order": 0.003078293568895903, "change_address": 0.001361767731259195, "report_damage": 0.0032776061174943907, "track_order": 0.9885665514218801}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.07090000400785357, "usage": {"input_tokens": 237, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "0fe0e99e631a0bd83e15e4200b8fc37631048951fabe74546a6a37164618eaac", "runtime": null} +{"task_id": "easy-intent-01", "family": "intent", "split": "public", "group": null, "ts": 1790809199.6157606, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cancel_order", "ordinal_ev": null, "probs": {"billing_question": 0.008350062041863383, "cancel_order": 0.9827395556133772, "change_address": 0.002380980312875335, "report_damage": 0.003060149037375123, "track_order": 0.003469252994509084}, "probs_as_returned": {"billing_question": 0.008350062041863383, "cancel_order": 0.9827395556133772, "change_address": 0.002380980312875335, "report_damage": 0.003060149037375123, "track_order": 0.003469252994509084}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05324758304050192, "usage": {"input_tokens": 237, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "63e59d92ddd8b352463b34609f87be90c8e8cd5cd206b3f3056806baa66af460", "runtime": null} +{"task_id": "easy-intent-02", "family": "intent", "split": "public", "group": null, "ts": 1790809199.6796355, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "change_address", "ordinal_ev": null, "probs": {"billing_question": 0.006453941294493193, "cancel_order": 0.004160046017396561, "change_address": 0.976248827903928, "report_damage": 0.0053466891642593726, "track_order": 0.007790495619922923}, "probs_as_returned": {"billing_question": 0.006453941294493193, "cancel_order": 0.004160046017396561, "change_address": 0.976248827903928, "report_damage": 0.0053466891642593726, "track_order": 0.007790495619922923}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05359842098550871, "usage": {"input_tokens": 239, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ee37d0f3168f243ed76d9cb6a14c2dce6ca8262759123f415a4c6070c4538c21", "runtime": null} +{"task_id": "easy-intent-03", "family": "intent", "split": "public", "group": null, "ts": 1790809199.7434382, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "report_damage", "ordinal_ev": null, "probs": {"billing_question": 0.0032856469914429327, "cancel_order": 0.0024009737772375207, "change_address": 0.0014534960934417126, "report_damage": 0.9909917808872951, "track_order": 0.0018681022505827403}, "probs_as_returned": {"billing_question": 0.0032856469914429327, "cancel_order": 0.0024009737772375207, "change_address": 0.0014534960934417126, "report_damage": 0.9909917808872951, "track_order": 0.0018681022505827403}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05331230798037723, "usage": {"input_tokens": 228, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8c17a5d4496d815b9fb8aefef96c5b406eb7bf711d51be89d22ca13253d1d28d", "runtime": null} +{"task_id": "easy-intent-04", "family": "intent", "split": "public", "group": null, "ts": 1790809199.806668, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "billing_question", "ordinal_ev": null, "probs": {"billing_question": 0.9920142822722872, "cancel_order": 0.005433032876037771, "change_address": 0.0007769531103232375, "report_damage": 0.0011320744243103107, "track_order": 0.000643657317041507}, "probs_as_returned": {"billing_question": 0.9920142822722872, "cancel_order": 0.005433032876037771, "change_address": 0.0007769531103232375, "report_damage": 0.0011320744243103107, "track_order": 0.000643657317041507}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05230753100477159, "usage": {"input_tokens": 233, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d8f70459feb07c644dc043ea0ebb040ec1f472a2bba6aef6f143624ebed01f3d", "runtime": null} +{"task_id": "easy-intent-05", "family": "intent", "split": "public", "group": null, "ts": 1790809199.878503, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "track_order", "ordinal_ev": null, "probs": {"billing_question": 0.005069572552208189, "cancel_order": 0.003479294303948175, "change_address": 0.0023878717693975727, "report_damage": 0.003479294303948175, "track_order": 0.9855839670704978}, "probs_as_returned": {"billing_question": 0.005069572552208189, "cancel_order": 0.003479294303948175, "change_address": 0.0023878717693975727, "report_damage": 0.003479294303948175, "track_order": 0.9855839670704978}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.061968157009687275, "usage": {"input_tokens": 232, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "aeabfc617501e3b52b685671fbcd89e397e37b8f1feb546c000c78060f3447d2", "runtime": null} +{"task_id": "easy-intent-06", "family": "intent", "split": "public", "group": null, "ts": 1790809199.9401288, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_alarm", "ordinal_ev": null, "probs": {"play_music": 0.006093371416719093, "send_message": 0.003927632182349683, "set_alarm": 0.9813860625238946, "turn_off_lights": 0.005722831872794024, "weather": 0.0028701020042427037}, "probs_as_returned": {"play_music": 0.006093371416719093, "send_message": 0.003927632182349683, "set_alarm": 0.9813860625238946, "turn_off_lights": 0.005722831872794024, "weather": 0.0028701020042427037}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05400926404399797, "usage": {"input_tokens": 225, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e5baa73df0ede130fe152f8fc82a4afc55007b2042774c2add8bef33e938a2ea", "runtime": null} +{"task_id": "easy-intent-07", "family": "intent", "split": "public", "group": null, "ts": 1790809200.0016422, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "play_music", "ordinal_ev": null, "probs": {"play_music": 0.9889646607601771, "send_message": 0.00653800507921397, "set_alarm": 0.0017509134460344028, "turn_off_lights": 0.0017509134460344028, "weather": 0.000995507268540069}, "probs_as_returned": {"play_music": 0.9889646607601771, "send_message": 0.00653800507921397, "set_alarm": 0.0017509134460344028, "turn_off_lights": 0.0017509134460344028, "weather": 0.000995507268540069}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05315710004651919, "usage": {"input_tokens": 218, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a378c077ce1a4cb5c055b1557dffa638392fc30bc27ff4faa4887aacd62a9ca8", "runtime": null} +{"task_id": "easy-intent-08", "family": "intent", "split": "public", "group": null, "ts": 1790809200.0676205, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "weather", "ordinal_ev": null, "probs": {"play_music": 0.0073499617445077865, "send_message": 0.00417892749053501, "set_alarm": 0.0030537350538425087, "turn_off_lights": 0.0047375981872448125, "weather": 0.9806797775238698}, "probs_as_returned": {"play_music": 0.0073499617445077865, "send_message": 0.00417892749053501, "set_alarm": 0.0030537350538425087, "turn_off_lights": 0.0047375981872448125, "weather": 0.9806797775238698}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05500068695982918, "usage": {"input_tokens": 220, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "78b1654145b5779f90804fca798f2982e5ba0b0d09051302a094d9b0a9f528cc", "runtime": null} +{"task_id": "easy-intent-09", "family": "intent", "split": "public", "group": null, "ts": 1790809200.1330826, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "send_message", "ordinal_ev": null, "probs": {"play_music": 0.012056092798137684, "send_message": 0.9738121212463231, "set_alarm": 0.004418343519024835, "turn_off_lights": 0.004704420988497517, "weather": 0.005009021448016902}, "probs_as_returned": {"play_music": 0.012056092798137684, "send_message": 0.9738121212463231, "set_alarm": 0.004418343519024835, "turn_off_lights": 0.004704420988497517, "weather": 0.005009021448016902}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054030488012358546, "usage": {"input_tokens": 223, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "be3460e73136f4919cc6b3846ebc4860e56a49321271513057764bfe656889a4", "runtime": null} +{"task_id": "easy-intent-10", "family": "intent", "split": "public", "group": null, "ts": 1790809200.1994722, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "turn_off_lights", "ordinal_ev": null, "probs": {"play_music": 0.0037055333570594024, "send_message": 0.0034801987605495434, "set_alarm": 0.0032685666377679874, "turn_off_lights": 0.9858401678875638, "weather": 0.0037055333570594024}, "probs_as_returned": {"play_music": 0.0037055333570594024, "send_message": 0.0034801987605495434, "set_alarm": 0.0032685666377679874, "turn_off_lights": 0.9858401678875638, "weather": 0.0037055333570594024}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05445742700248957, "usage": {"input_tokens": 221, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "4221fd040951aea1ff95eee1af6222195d2b7d940e557b0eb83dc328d7454d6e", "runtime": null} +{"task_id": "easy-intent-11", "family": "intent", "split": "public", "group": null, "ts": 1790809200.2655914, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_alarm", "ordinal_ev": null, "probs": {"play_music": 0.00611383130526039, "send_message": 0.0027046210473488227, "set_alarm": 0.9846812885886964, "turn_off_lights": 0.004756929120067906, "weather": 0.0017433299386264444}, "probs_as_returned": {"play_music": 0.00611383130526039, "send_message": 0.0027046210473488227, "set_alarm": 0.9846812885886964, "turn_off_lights": 0.004756929120067906, "weather": 0.0017433299386264444}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05490372603526339, "usage": {"input_tokens": 221, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8b0613ae30f39c34190e3dce27f0e6d8fd1f27d9c1ff1a08a1c6d3d9ed44168d", "runtime": null} +{"task_id": "easy-fact-00", "family": "fact", "split": "public", "group": null, "ts": 1790809200.3313708, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9710643566281367, "no": 0.028935643371863273}, "probs_as_returned": {"yes": 0.9710643566281367, "no": 0.028935643371863273}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054723599983844906, "usage": {"input_tokens": 199, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "12c93390622963713919dd9d68014aeb1d9ee588fd6d292386fa4de90685c3a6", "runtime": null} +{"task_id": "easy-fact-01", "family": "fact", "split": "public", "group": null, "ts": 1790809200.3980494, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.010802420461426548, "no": 0.9891975795385735}, "probs_as_returned": {"yes": 0.010802420461426548, "no": 0.9891975795385735}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05640265700640157, "usage": {"input_tokens": 197, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "dcf5e95244f20fff36abb6b0d4c646261795ca47d4c0fdc0e1ca6006697db20f", "runtime": null} +{"task_id": "easy-fact-02", "family": "fact", "split": "public", "group": null, "ts": 1790809200.4626622, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9743891388556707, "no": 0.02561086114432931}, "probs_as_returned": {"yes": 0.9743891388556707, "no": 0.02561086114432931}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05494501901557669, "usage": {"input_tokens": 205, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "06f1defc881be6f6913695f67aea44079a44f63598d49fe4cf847c836348decf", "runtime": null} +{"task_id": "easy-fact-03", "family": "fact", "split": "public", "group": null, "ts": 1790809200.5293405, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.011493812770397228, "no": 0.9885061872296028}, "probs_as_returned": {"yes": 0.011493812770397228, "no": 0.9885061872296028}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054548991029150784, "usage": {"input_tokens": 208, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ac7b9c8de6f82c770a8781ffb74872c4463bc44ea55dea15e1188bbdaa66b205", "runtime": null} +{"task_id": "easy-fact-04", "family": "fact", "split": "public", "group": null, "ts": 1790809200.5974357, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9710643566281367, "no": 0.028935643371863273}, "probs_as_returned": {"yes": 0.9710643566281367, "no": 0.028935643371863273}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05654464696999639, "usage": {"input_tokens": 197, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "05ead01f33a1b6e07f8eb1ba4ed9d0db4c6168a4d57043f5d06035aae5acbb52", "runtime": null} +{"task_id": "easy-fact-05", "family": "fact", "split": "public", "group": null, "ts": 1790809200.663285, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.07967467169628852, "no": 0.9203253283037115}, "probs_as_returned": {"yes": 0.07967467169628852, "no": 0.9203253283037115}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05427099298685789, "usage": {"input_tokens": 198, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c55c922b96a10bf910ac6bc63a56bae337cc2ea183ec8ac42db6efe82d9f3438", "runtime": null} +{"task_id": "easy-fact-06", "family": "fact", "split": "public", "group": null, "ts": 1790809200.727296, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9608203271108413, "no": 0.03917967288915869}, "probs_as_returned": {"yes": 0.9608203271108413, "no": 0.03917967288915869}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05310487397946417, "usage": {"input_tokens": 218, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "5dff03b653be30a2b129ec60d53538ffa8ab168bb9164107071af39c6dafe71d", "runtime": null} +{"task_id": "easy-fact-07", "family": "fact", "split": "public", "group": null, "ts": 1790809200.7903893, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.016659759016074603, "no": 0.9833402409839254}, "probs_as_returned": {"yes": 0.016659759016074603, "no": 0.9833402409839254}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052477571996860206, "usage": {"input_tokens": 193, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "bdc9685fc6a15a3a301b8270800ee6ec0151ad362544be38dfcf02a29fa4fb75", "runtime": null} +{"task_id": "easy-fact-08", "family": "fact", "split": "public", "group": null, "ts": 1790809200.8536887, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9759090252955993, "no": 0.02409097470440069}, "probs_as_returned": {"yes": 0.9759090252955993, "no": 0.02409097470440069}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053135246969759464, "usage": {"input_tokens": 189, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "648e0c0b5d6cb0fbc995359e919d0f16e4693f32db734c67c58a3874aec41943", "runtime": null} +{"task_id": "easy-fact-09", "family": "fact", "split": "public", "group": null, "ts": 1790809200.9154594, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.013841134222312299, "no": 0.9861588657776877}, "probs_as_returned": {"yes": 0.013841134222312299, "no": 0.9861588657776877}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05241246899822727, "usage": {"input_tokens": 189, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ad6a895bf979c4529d64b6fcd128bf4251335c7b37506dfbb2fa47d1d528fd5c", "runtime": null} +{"task_id": "easy-fact-10", "family": "fact", "split": "public", "group": null, "ts": 1790809200.9823, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9405016353866033, "no": 0.05949836461339675}, "probs_as_returned": {"yes": 0.9405016353866033, "no": 0.05949836461339675}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.058301462966483086, "usage": {"input_tokens": 197, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9bed56acdb0fb24046f6e783e2ce37a9afb585c67075b74c922e6173abb3a3c4", "runtime": null} +{"task_id": "easy-fact-11", "family": "fact", "split": "public", "group": null, "ts": 1790809201.0447483, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.012228909296002857, "no": 0.9877710907039972}, "probs_as_returned": {"yes": 0.012228909296002857, "no": 0.9877710907039972}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05360094696516171, "usage": {"input_tokens": 198, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "3cf94ba00145cd37a871038936af79a201456e928b9630e59858554ed03a4f59", "runtime": null} +{"task_id": "easy-extraction-00", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.105547, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "paypal", "ordinal_ev": null, "probs": {"bank_transfer": 0.005387207125398065, "cash": 0.004462965891172113, "credit_card": 0.006502850175498755, "paypal": 0.9836469768079311}, "probs_as_returned": {"bank_transfer": 0.005387207125398065, "cash": 0.004462965891172113, "credit_card": 0.006502850175498755, "paypal": 0.9836469768079311}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05292710999492556, "usage": {"input_tokens": 191, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "34bd31e90803f5c9298cf0a07f0951af501193be3bc3cee4f735e0e54e797235", "runtime": null} +{"task_id": "easy-extraction-01", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.1686747, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cash", "ordinal_ev": null, "probs": {"bank_transfer": 0.005362202749492144, "cash": 0.9790814538511988, "credit_card": 0.011384226924270966, "paypal": 0.004172116475037995}, "probs_as_returned": {"bank_transfer": 0.005362202749492144, "cash": 0.9790814538511988, "credit_card": 0.011384226924270966, "paypal": 0.004172116475037995}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05316750198835507, "usage": {"input_tokens": 193, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1093fd545ce8f9934b8793a34f16c4af8d84f597e755cf2423799a97376657f4", "runtime": null} +{"task_id": "easy-extraction-02", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.2302415, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "bank_transfer", "ordinal_ev": null, "probs": {"bank_transfer": 0.9886611045091759, "cash": 0.006959188878070115, "credit_card": 0.002395327053946649, "paypal": 0.001984379558807392}, "probs_as_returned": {"bank_transfer": 0.9886611045091759, "cash": 0.006959188878070115, "credit_card": 0.002395327053946649, "paypal": 0.001984379558807392}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05336353601887822, "usage": {"input_tokens": 194, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "5674a2e1d974db55735d9b17f51b3ffe800df951eba947b70bc20c3a254f95ee", "runtime": null} +{"task_id": "easy-extraction-03", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.2917578, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "high", "ordinal_ev": null, "probs": {"high": 0.9233443241587275, "low": 0.012959506894251118, "medium": 0.01217143488730339, "urgent": 0.051524734059717935}, "probs_as_returned": {"high": 0.9233443241587275, "low": 0.012959506894251118, "medium": 0.01217143488730339, "urgent": 0.051524734059717935}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052858371986076236, "usage": {"input_tokens": 196, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "3e4346f178e5077ac3a665b387dbfd209a300c941ff9e0256ff19972dd724b35", "runtime": null} +{"task_id": "easy-extraction-04", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.354225, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "low", "ordinal_ev": null, "probs": {"high": 0.012079556291950424, "low": 0.9757073270220792, "medium": 0.007786174264223241, "urgent": 0.004426942421747175}, "probs_as_returned": {"high": 0.012079556291950424, "low": 0.9757073270220792, "medium": 0.007786174264223241, "urgent": 0.004426942421747175}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05369667295599356, "usage": {"input_tokens": 194, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "936516e3df44303bbdc07fd3a8a0a527b1febbdbb49b842482cc98685ce59a13", "runtime": null} +{"task_id": "easy-extraction-05", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.414276, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "urgent", "ordinal_ev": null, "probs": {"high": 0.012123545380943897, "low": 0.004172879641756854, "medium": 0.004443063766191328, "urgent": 0.9792605112111079}, "probs_as_returned": {"high": 0.012123545380943897, "low": 0.004172879641756854, "medium": 0.004443063766191328, "urgent": 0.9792605112111079}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05233745800796896, "usage": {"input_tokens": 194, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "220910348688ea3a4e0a3455f927088fa1f4a889ec9c5a225b7c0b266107c9bb", "runtime": null} +{"task_id": "easy-extraction-06", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.4746969, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "M", "ordinal_ev": null, "probs": {"L": 0.011362559593985683, "M": 0.9772179520276576, "S": 0.006067491605402367, "XL": 0.005351996772954305}, "probs_as_returned": {"L": 0.011362559593985683, "M": 0.9772179520276576, "S": 0.006067491605402367, "XL": 0.005351996772954305}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052841529017314315, "usage": {"input_tokens": 187, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7adef6d8f6cfc658c765ad7d1e70cb0b4e69e1954fd724f23cb2ef738a10968a", "runtime": null} +{"task_id": "easy-extraction-07", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.5374498, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "XL", "ordinal_ev": null, "probs": {"L": 0.0236463954766581, "M": 0.009227085930027284, "S": 0.009227085930027284, "XL": 0.9578994326632874}, "probs_as_returned": {"L": 0.0236463954766581, "M": 0.009227085930027284, "S": 0.009227085930027284, "XL": 0.9578994326632874}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05306885595200583, "usage": {"input_tokens": 184, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e3ebfe1c1ceb07adc48bfecad43adcc800494fda8bbec1a689e3560c4f1f4275", "runtime": null} +{"task_id": "easy-extraction-08", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.5998538, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "thursday", "ordinal_ev": null, "probs": {"friday": 0.005378595700972278, "monday": 0.0030580785963976943, "thursday": 0.9820746521827389, "tuesday": 0.0047443367599455805, "wednesday": 0.0047443367599455805}, "probs_as_returned": {"friday": 0.005378595700972278, "monday": 0.0030580785963976943, "thursday": 0.9820746521827389, "tuesday": 0.0047443367599455805, "wednesday": 0.0047443367599455805}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053394540038425475, "usage": {"input_tokens": 195, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "47f5e407bb272f086cefabbb08935544890df695106375cab838cc494401c1d4", "runtime": null} +{"task_id": "easy-extraction-09", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.6635482, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "monday", "ordinal_ev": null, "probs": {"friday": 0.006487826443252909, "monday": 0.9813744363156989, "thursday": 0.00223308577898157, "tuesday": 0.004181887565710403, "wednesday": 0.005722763896356328}, "probs_as_returned": {"friday": 0.006487826443252909, "monday": 0.9813744363156989, "thursday": 0.00223308577898157, "tuesday": 0.004181887565710403, "wednesday": 0.005722763896356328}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053598670987412333, "usage": {"input_tokens": 197, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "78d52b5f6e068e149867d2bdb7ee1467444c7d32f6f6320d60462e4568c4ef0d", "runtime": null} +{"task_id": "easy-extraction-10", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.7260056, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "friday", "ordinal_ev": null, "probs": {"friday": 0.9843070496345583, "monday": 0.005390822043802262, "thursday": 0.005063004435363676, "tuesday": 0.0019756402868039927, "wednesday": 0.0032634835994717003}, "probs_as_returned": {"friday": 0.9843070496345583, "monday": 0.005390822043802262, "thursday": 0.005063004435363676, "tuesday": 0.0019756402868039927, "wednesday": 0.0032634835994717003}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05365842999890447, "usage": {"input_tokens": 192, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "86e76a9f1d1a410206b630d61436593c67d6a756f6524b09a75f10979265dc14", "runtime": null} +{"task_id": "easy-extraction-11", "family": "extraction", "split": "public", "group": null, "ts": 1790809201.78623, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "credit_card", "ordinal_ev": null, "probs": {"bank_transfer": 0.00782696473030407, "cash": 0.004450134632367189, "credit_card": 0.9808189129742555, "paypal": 0.006903987663073161}, "probs_as_returned": {"bank_transfer": 0.00782696473030407, "cash": 0.004450134632367189, "credit_card": 0.9808189129742555, "paypal": 0.006903987663073161}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05294459901051596, "usage": {"input_tokens": 198, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f4f2ee4aaf356359a4eade39f16a3c272fab4e78f0c90339001f0f9c42c69d64", "runtime": null} +{"task_id": "easy-tool_selection-00", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809201.8485699, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "get_weather", "ordinal_ev": null, "probs": {"convert_currency": 0.007829807987743406, "create_calendar_event": 0.0032530999253130594, "get_weather": 0.9811752060057245, "send_email": 0.0050468950765359955, "translate_text": 0.0026949910046831105}, "probs_as_returned": {"convert_currency": 0.007829807987743406, "create_calendar_event": 0.0032530999253130594, "get_weather": 0.9811752060057245, "send_email": 0.0050468950765359955, "translate_text": 0.0026949910046831105}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05388889496680349, "usage": {"input_tokens": 227, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7a64565c1ea6e9286acfb0c26dce3f57250197b68d8667d58612d86035e4e000", "runtime": null} +{"task_id": "easy-tool_selection-01", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809201.9116292, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "send_email", "ordinal_ev": null, "probs": {"convert_currency": 0.004770803588408859, "create_calendar_event": 0.0034862463985914503, "get_weather": 0.001642093020855755, "send_email": 0.9875532959266949, "translate_text": 0.0025475610654490583}, "probs_as_returned": {"convert_currency": 0.004770803588408859, "create_calendar_event": 0.0034862463985914503, "get_weather": 0.001642093020855755, "send_email": 0.9875532959266949, "translate_text": 0.0025475610654490583}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05310086498502642, "usage": {"input_tokens": 229, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "60a3781266e735741195583056e7f42cab63670fceb1bc256a90749aeee61580", "runtime": null} +{"task_id": "easy-tool_selection-02", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809201.9735434, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "create_calendar_event", "ordinal_ev": null, "probs": {"convert_currency": 0.006516607444799637, "create_calendar_event": 0.9857279498146715, "get_weather": 0.001978492219286686, "send_email": 0.0030694545003598877, "translate_text": 0.002707496020882359}, "probs_as_returned": {"convert_currency": 0.006516607444799637, "create_calendar_event": 0.9857279498146715, "get_weather": 0.001978492219286686, "send_email": 0.0030694545003598877, "translate_text": 0.002707496020882359}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05315082200104371, "usage": {"input_tokens": 232, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "fe23c5b1c0f7c7af4b9b0b990b6125ef959ba4cff556e1661a6d988a497b1f04", "runtime": null} +{"task_id": "easy-tool_selection-03", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.034381, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "convert_currency", "ordinal_ev": null, "probs": {"convert_currency": 0.9879132695583835, "create_calendar_event": 0.0050815538345912745, "get_weather": 0.0017490519673023376, "send_email": 0.0015427989590568122, "translate_text": 0.003713325680665897}, "probs_as_returned": {"convert_currency": 0.9879132695583835, "create_calendar_event": 0.0050815538345912745, "get_weather": 0.0017490519673023376, "send_email": 0.0015427989590568122, "translate_text": 0.003713325680665897}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052334961015731096, "usage": {"input_tokens": 229, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ea727bac8863ba34e3dea0be8a59df78673a3d9912d43270e07f17fb3de21c0c", "runtime": null} +{"task_id": "easy-tool_selection-04", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.0977094, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "translate_text", "ordinal_ev": null, "probs": {"convert_currency": 0.0065024668896828735, "create_calendar_event": 0.004462702891725051, "get_weather": 0.002383038386749453, "send_email": 0.0030627940173430794, "translate_text": 0.9835889978144995}, "probs_as_returned": {"convert_currency": 0.0065024668896828735, "create_calendar_event": 0.004462702891725051, "get_weather": 0.002383038386749453, "send_email": 0.0030627940173430794, "translate_text": 0.9835889978144995}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053545792994555086, "usage": {"input_tokens": 226, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9139c5d2c907c33865dd90599d172bfe3901a85f25c67d6d51b4eb656b8f7f9b", "runtime": null} +{"task_id": "easy-tool_selection-05", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.1604862, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "create_calendar_event", "ordinal_ev": null, "probs": {"convert_currency": 0.007354093936894176, "create_calendar_event": 0.9812311228123023, "get_weather": 0.0022327596869927153, "send_email": 0.006486878942590166, "translate_text": 0.002695144621220672}, "probs_as_returned": {"convert_currency": 0.007354093936894176, "create_calendar_event": 0.9812311228123023, "get_weather": 0.0022327596869927153, "send_email": 0.006486878942590166, "translate_text": 0.002695144621220672}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05288134701550007, "usage": {"input_tokens": 232, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d06b0b139bc420dd03a3951d77dd93fa058fd952cd752854f11d47d23ff24089", "runtime": null} +{"task_id": "easy-tool_selection-06", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.2244015, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "search_flights", "ordinal_ev": null, "probs": {"book_restaurant": 0.002723028128906097, "call_taxi": 0.0022558595435807336, "get_stock_price": 0.0019898423713477923, "search_flights": 0.9913828092209783, "set_reminder": 0.0016484607351869884}, "probs_as_returned": {"book_restaurant": 0.002723028128906097, "call_taxi": 0.0022558595435807336, "get_stock_price": 0.0019898423713477923, "search_flights": 0.9913828092209783, "set_reminder": 0.0016484607351869884}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05422422802075744, "usage": {"input_tokens": 236, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a8315ca687a7d5d4247d9bfef8bcdb6216f524e07fcc0ae0daeee1b43f6446b9", "runtime": null} +{"task_id": "easy-tool_selection-07", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.2886288, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "book_restaurant", "ordinal_ev": null, "probs": {"book_restaurant": 0.9875027503742834, "call_taxi": 0.005758500527357331, "get_stock_price": 0.0010583971646233432, "search_flights": 0.0011998917646541929, "set_reminder": 0.004480460169081839}, "probs_as_returned": {"book_restaurant": 0.9875027503742834, "call_taxi": 0.005758500527357331, "get_stock_price": 0.0010583971646233432, "search_flights": 0.0011998917646541929, "set_reminder": 0.004480460169081839}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054042228031903505, "usage": {"input_tokens": 236, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "17225209afce500f3fb78eafded9cdd217cc92765deb5e4b550c2ea13a0d5552", "runtime": null} +{"task_id": "easy-tool_selection-08", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.3514938, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_reminder", "ordinal_ev": null, "probs": {"book_restaurant": 0.0027204340659280554, "call_taxi": 0.0022537104865137892, "get_stock_price": 0.0018670590411967879, "search_flights": 0.0027204340659280554, "set_reminder": 0.9904383623404333}, "probs_as_returned": {"book_restaurant": 0.0027204340659280554, "call_taxi": 0.0022537104865137892, "get_stock_price": 0.0018670590411967879, "search_flights": 0.0027204340659280554, "set_reminder": 0.9904383623404333}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053362559992820024, "usage": {"input_tokens": 235, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "3612b0bf61557578bbdd33701911b4066508f84e258f1b6144b860bd9f6d6c97", "runtime": null} +{"task_id": "easy-tool_selection-09", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.413625, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "get_stock_price", "ordinal_ev": null, "probs": {"book_restaurant": 0.006098491673709373, "call_taxi": 0.0030585022990139332, "get_stock_price": 0.98221073526815, "search_flights": 0.006098491673709373, "set_reminder": 0.0025337790854174126}, "probs_as_returned": {"book_restaurant": 0.006098491673709373, "call_taxi": 0.0030585022990139332, "get_stock_price": 0.98221073526815, "search_flights": 0.006098491673709373, "set_reminder": 0.0025337790854174126}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05303042399464175, "usage": {"input_tokens": 231, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2bbfa079ef8e8df31b7dd11a5a0c984d4d3c3438e36f5e7e70798ad3c14c7932", "runtime": null} +{"task_id": "easy-tool_selection-10", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.476455, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "call_taxi", "ordinal_ev": null, "probs": {"book_restaurant": 0.008868475619137011, "call_taxi": 0.9802820079262876, "get_stock_price": 0.002230600011225041, "search_flights": 0.0053687778295083005, "set_reminder": 0.0032501386138420975}, "probs_as_returned": {"book_restaurant": 0.008868475619137011, "call_taxi": 0.9802820079262876, "get_stock_price": 0.002230600011225041, "search_flights": 0.0053687778295083005, "set_reminder": 0.0032501386138420975}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05289635399822146, "usage": {"input_tokens": 232, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1256893414eb00c914623a0d3fa5c96cc0f225c3757380dd480b10b30bd56705", "runtime": null} +{"task_id": "easy-tool_selection-11", "family": "tool_selection", "split": "public", "group": null, "ts": 1790809202.5399096, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "set_reminder", "ordinal_ev": null, "probs": {"book_restaurant": 0.0024029836241437775, "call_taxi": 0.0018696659764764975, "get_stock_price": 0.0016491898330211032, "search_flights": 0.002256857361858334, "set_reminder": 0.9918213032045002}, "probs_as_returned": {"book_restaurant": 0.0024029836241437775, "call_taxi": 0.0018696659764764975, "get_stock_price": 0.0016491898330211032, "search_flights": 0.002256857361858334, "set_reminder": 0.9918213032045002}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05290257098386064, "usage": {"input_tokens": 235, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "40743fc69eef2cfb3c4da8ee1167c0321ff89549f5fadfead84e1440bf71d8db", "runtime": null} +{"task_id": "original-policy-01-0", "family": "policy", "split": "public", "group": "original-policy-01", "ts": 1790809202.6063561, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.022659190258890113, "no": 0.9773408097411099}, "probs_as_returned": {"yes": 0.022659190258890113, "no": 0.9773408097411099}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054190903960261494, "usage": {"input_tokens": 219, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a276741f0eb42fc6dba9d94501ad6d7e9d456550ee0894e2d270bd58352c85c5", "runtime": null} +{"task_id": "original-policy-01-1", "family": "policy", "split": "public", "group": "original-policy-01", "ts": 1790809202.6723044, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.005797568831524007, "no": 0.994202431168476}, "probs_as_returned": {"yes": 0.005797568831524007, "no": 0.994202431168476}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05371244502021, "usage": {"input_tokens": 221, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "aef19c86a2ec5682cfc80ffed0f50097d5c2f4b516c3c123459639580ed650cf", "runtime": null} +{"task_id": "original-policy-02-0", "family": "policy", "split": "public", "group": "original-policy-02", "ts": 1790809202.7362373, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9652800911770798, "no": 0.0347199088229202}, "probs_as_returned": {"yes": 0.9652800911770798, "no": 0.0347199088229202}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053028328984510154, "usage": {"input_tokens": 208, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9c6d02d3436485f99e266b5efcca67d87bdd0c0448a88391d8169edf1558afbe", "runtime": null} +{"task_id": "original-policy-02-1", "family": "policy", "split": "public", "group": "original-policy-02", "ts": 1790809202.801814, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.91062577829021, "no": 0.08937422170979004}, "probs_as_returned": {"yes": 0.91062577829021, "no": 0.08937422170979004}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05453022598521784, "usage": {"input_tokens": 206, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "34220d159a46699ac63f901fac0169efe53003e06da711217ff70292083b26b8", "runtime": null} +{"task_id": "original-policy-03-0", "family": "policy", "split": "public", "group": "original-policy-03", "ts": 1790809202.864749, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8879876242264044, "no": 0.11201237577359557}, "probs_as_returned": {"yes": 0.8879876242264044, "no": 0.11201237577359557}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05364256398752332, "usage": {"input_tokens": 210, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "fcd872c07ed1e7db38f0e59dae74a9f5042e8e6a0bd8805a0f68e8ca2dec1c00", "runtime": null} +{"task_id": "original-policy-03-1", "family": "policy", "split": "public", "group": "original-policy-03", "ts": 1790809202.9261985, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7557061813028185, "no": 0.24429381869718148}, "probs_as_returned": {"yes": 0.7557061813028185, "no": 0.24429381869718148}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05279253504704684, "usage": {"input_tokens": 209, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7e4a323e350f67e13e5f4aaa64933cec5c79c488908e9fd024d2e2f133f65ea5", "runtime": null} +{"task_id": "original-policy-04-0", "family": "policy", "split": "public", "group": "original-policy-04", "ts": 1790809202.9887743, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.09461348765523125, "no": 0.9053865123447687}, "probs_as_returned": {"yes": 0.09461348765523125, "no": 0.9053865123447687}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053309708018787205, "usage": {"input_tokens": 216, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "721d666ea0b20279d491dfc298252cef92b3ccf3d557926f11bb8964d8b2e337", "runtime": null} +{"task_id": "original-policy-04-1", "family": "policy", "split": "public", "group": "original-policy-04", "ts": 1790809203.0532064, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.012228909296002857, "no": 0.9877710907039972}, "probs_as_returned": {"yes": 0.012228909296002857, "no": 0.9877710907039972}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054208226036280394, "usage": {"input_tokens": 212, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9d0029574fbb8aaeb9464050b240c6bf4cf03b1c0330567ef74cdad40ea53c0a", "runtime": null} +{"task_id": "original-policy-05-0", "family": "policy", "split": "public", "group": "original-policy-05", "ts": 1790809203.1167583, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.03917967288915878, "no": 0.9608203271108412}, "probs_as_returned": {"yes": 0.03917967288915878, "no": 0.9608203271108412}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05344612500630319, "usage": {"input_tokens": 220, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "59c5a6a8131812f89b458dfb3fbd660d41f87a795155c5950c981f6e83419c33", "runtime": null} +{"task_id": "original-policy-05-1", "family": "policy", "split": "public", "group": "original-policy-05", "ts": 1790809203.177957, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.04979902705524531, "no": 0.9502009729447547}, "probs_as_returned": {"yes": 0.04979902705524531, "no": 0.9502009729447547}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05248717003269121, "usage": {"input_tokens": 212, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8c1b47f1cbd449319e7c721332b3f8f013ab449392c764a5e018e564804387d9", "runtime": null} +{"task_id": "original-policy-06-0", "family": "policy", "split": "public", "group": "original-policy-06", "ts": 1790809203.240201, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.91062577829021, "no": 0.08937422170979004}, "probs_as_returned": {"yes": 0.91062577829021, "no": 0.08937422170979004}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05370797700015828, "usage": {"input_tokens": 222, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "5f0bf8b1d887c9bd22d19d0c47830444a7adae23c7862823cf441e5cc8d68c8a", "runtime": null} +{"task_id": "original-policy-06-1", "family": "policy", "split": "public", "group": "original-policy-06", "ts": 1790809203.302127, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8604937671763584, "no": 0.1395062328236416}, "probs_as_returned": {"yes": 0.8604937671763584, "no": 0.1395062328236416}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05304385395720601, "usage": {"input_tokens": 211, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f05344b9bfad91abeba3b18c07deba6bfbf8d407918d333f1cddd5010a5f8cba", "runtime": null} +{"task_id": "original-intent-01-0", "family": "intent", "split": "public", "group": "original-intent-01", "ts": 1790809203.3647604, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "refund", "ordinal_ev": null, "probs": {"cancel": 0.019858206552561444, "change_address": 0.007277679357392087, "other": 0.11503962460331064, "refund": 0.8565281796860903, "status": 0.0012963098006456009}, "probs_as_returned": {"cancel": 0.019858206552561444, "change_address": 0.007277679357392087, "other": 0.11503962460331064, "refund": 0.8565281796860903, "status": 0.0012963098006456009}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05289072298910469, "usage": {"input_tokens": 216, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "fff65c1256c5aecee8f26b495cd70be2349fc11a1c80933e323720557b204cc5", "runtime": null} +{"task_id": "original-intent-01-1", "family": "intent", "split": "public", "group": "original-intent-01", "ts": 1790809203.42796, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "refund", "ordinal_ev": null, "probs": {"cancel": 0.013157397907281692, "change_address": 0.010900086144070705, "other": 0.19533400388242703, "refund": 0.7766138243051219, "status": 0.003994687761098555}, "probs_as_returned": {"cancel": 0.013157397907281692, "change_address": 0.010900086144070705, "other": 0.19533400388242703, "refund": 0.7766138243051219, "status": 0.003994687761098555}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05378658202243969, "usage": {"input_tokens": 218, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "fb52816dbeaeecd8781c257bfd4adfb3615540d2ab0f9876f12ca3f99f38fcb4", "runtime": null} +{"task_id": "original-intent-02-0", "family": "intent", "split": "public", "group": "original-intent-02", "ts": 1790809203.4902308, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "other", "ordinal_ev": null, "probs": {"cancel": 0.011108283739639322, "change_address": 0.017233512714200254, "other": 0.4791247461286767, "refund": 0.013408711288806937, "status": 0.4791247461286767}, "probs_as_returned": {"cancel": 0.011108283739639322, "change_address": 0.017233512714200254, "other": 0.4791247461286767, "refund": 0.013408711288806937, "status": 0.4791247461286767}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05274607602041215, "usage": {"input_tokens": 218, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e73189a5af92c08d80e3f4dcb06fe1d2ec2636c76d65800c6010c4fdb70a3e70", "runtime": null} +{"task_id": "original-intent-02-1", "family": "intent", "split": "public", "group": "original-intent-02", "ts": 1790809203.5513897, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.002321175467540561, "change_address": 0.0031764458100722866, "other": 0.0323645156724377, "refund": 0.004082518984442435, "status": 0.958055344065507}, "probs_as_returned": {"cancel": 0.002321175467540561, "change_address": 0.0031764458100722866, "other": 0.0323645156724377, "refund": 0.004082518984442435, "status": 0.958055344065507}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05259585496969521, "usage": {"input_tokens": 217, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "6c278fbc89c8d972d7a46af085434aad933c672f4154d9367c0fee30091234a8", "runtime": null} +{"task_id": "original-intent-03-0", "family": "intent", "split": "public", "group": "original-intent-03", "ts": 1790809203.6131387, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "change_address", "ordinal_ev": null, "probs": {"cancel": 0.009915815183017014, "change_address": 0.9080094929418733, "other": 0.04469398562198547, "refund": 0.011969292639071262, "status": 0.02541141361405287}, "probs_as_returned": {"cancel": 0.009915815183017014, "change_address": 0.9080094929418733, "other": 0.04469398562198547, "refund": 0.011969292639071262, "status": 0.02541141361405287}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05305930197937414, "usage": {"input_tokens": 218, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "87c3a2dda41a6eb00305a6db2f93795d55c8bda8c64886edee79531e4c4816fd", "runtime": null} +{"task_id": "original-intent-03-1", "family": "intent", "split": "public", "group": "original-intent-03", "ts": 1790809203.6756818, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "change_address", "ordinal_ev": null, "probs": {"cancel": 0.006265845660924066, "change_address": 0.947796734818206, "other": 0.03409105347210196, "refund": 0.008053162589058565, "status": 0.0037932034597093207}, "probs_as_returned": {"cancel": 0.006265845660924066, "change_address": 0.947796734818206, "other": 0.03409105347210196, "refund": 0.008053162589058565, "status": 0.0037932034597093207}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05302209098590538, "usage": {"input_tokens": 217, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1112825019689ae9ebbf69fcaf5f5b1f2e72097764d1648adc8a214970911137", "runtime": null} +{"task_id": "original-intent-04-0", "family": "intent", "split": "public", "group": "original-intent-04", "ts": 1790809203.7375238, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cancel", "ordinal_ev": null, "probs": {"cancel": 0.9378732275183023, "change_address": 0.004530802608425531, "other": 0.05233548212728289, "refund": 0.0042552831109369605, "status": 0.0010052046350524108}, "probs_as_returned": {"cancel": 0.9378732275183023, "change_address": 0.004530802608425531, "other": 0.05233548212728289, "refund": 0.0042552831109369605, "status": 0.0010052046350524108}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05315842299023643, "usage": {"input_tokens": 222, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e21cab18d338e3a3838c0e3e2cc0755b22da3fdb063f4b55957efac3985ecd4a", "runtime": null} +{"task_id": "original-intent-04-1", "family": "intent", "split": "public", "group": "original-intent-04", "ts": 1790809203.7991266, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cancel", "ordinal_ev": null, "probs": {"cancel": 0.947910801764849, "change_address": 0.004300825146805656, "other": 0.0438207144703226, "refund": 0.0029516958001635546, "status": 0.0010159628178592127}, "probs_as_returned": {"cancel": 0.947910801764849, "change_address": 0.004300825146805656, "other": 0.0438207144703226, "refund": 0.0029516958001635546, "status": 0.0010159628178592127}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05295716103864834, "usage": {"input_tokens": 217, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "603abe4242a061da701096409403f2af2dd17ffedbc6d5949feaa1d4fa857022", "runtime": null} +{"task_id": "original-intent-05-0", "family": "intent", "split": "public", "group": "original-intent-05", "ts": 1790809203.8615444, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "other", "ordinal_ev": null, "probs": {"cancel": 0.007151922435870813, "change_address": 0.004066329316103547, "other": 0.697318944283325, "refund": 0.2897195038147706, "status": 0.0017433001499300139}, "probs_as_returned": {"cancel": 0.007151922435870813, "change_address": 0.004066329316103547, "other": 0.697318944283325, "refund": 0.2897195038147706, "status": 0.0017433001499300139}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05332430801354349, "usage": {"input_tokens": 217, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d849911d59a3285dcddcef30f103671f5aa3488b5f13e1843e23dbd3fb3a75ff", "runtime": null} +{"task_id": "original-intent-05-1", "family": "intent", "split": "public", "group": "original-intent-05", "ts": 1790809203.9236293, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "other", "ordinal_ev": null, "probs": {"cancel": 0.008842063949217063, "change_address": 0.00367366817047693, "other": 0.8621092581823092, "refund": 0.1232862922745069, "status": 0.0020887174234899913}, "probs_as_returned": {"cancel": 0.008842063949217063, "change_address": 0.00367366817047693, "other": 0.8621092581823092, "refund": 0.1232862922745069, "status": 0.0020887174234899913}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053221766953356564, "usage": {"input_tokens": 216, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ac40cc3a526f7e159c06dc2aeef6e7595972980885d9a8ee19eeb68bbd3049f7", "runtime": null} +{"task_id": "original-intent-06-0", "family": "intent", "split": "public", "group": "original-intent-06", "ts": 1790809203.9882348, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.04041415591597284, "change_address": 0.015770051641193803, "other": 0.2065130791599608, "refund": 0.013064506583094595, "status": 0.7242382066997779}, "probs_as_returned": {"cancel": 0.04041415591597284, "change_address": 0.015770051641193803, "other": 0.2065130791599608, "refund": 0.013064506583094595, "status": 0.7242382066997779}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05412392795551568, "usage": {"input_tokens": 214, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "453b2e0f1ecd159b9fae1575e97ac62bd38f4247269e74f8c2ce43e79217fbd5", "runtime": null} +{"task_id": "original-intent-06-1", "family": "intent", "split": "public", "group": "original-intent-06", "ts": 1790809204.0524893, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "status", "ordinal_ev": null, "probs": {"cancel": 0.010751220386095738, "change_address": 0.0078564127691029, "other": 0.10288184565711354, "refund": 0.010097435369874964, "status": 0.868413085817813}, "probs_as_returned": {"cancel": 0.010751220386095738, "change_address": 0.0078564127691029, "other": 0.10288184565711354, "refund": 0.010097435369874964, "status": 0.868413085817813}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05330343497917056, "usage": {"input_tokens": 219, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "08d05d6104b5b8623b40f71e5854dfbf4e9234a72444ebac02b6d168301e41a8", "runtime": null} +{"task_id": "original-ordinal-01-0", "family": "ordinal", "split": "public", "group": "original-ordinal-01", "ts": 1790809204.1163173, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.07628057138193826, "probs": {"0": 0.9336912617650633, "1": 0.05906752177300131, "2": 0.004510599776869139, "3": 0.0027306166850662277}, "probs_as_returned": {"0": 0.9336912617650633, "1": 0.05906752177300131, "2": 0.004510599776869139, "3": 0.0027306166850662277}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054263721976894885, "usage": {"input_tokens": 226, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "37654240c3b27171606c1680a2b293058317c29ea4d918f5f2bd300ce0921189", "runtime": null} +{"task_id": "original-ordinal-01-1", "family": "ordinal", "split": "public", "group": "original-ordinal-01", "ts": 1790809204.1792514, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.043562903333643106, "probs": {"0": 0.9617730521586384, "1": 0.03459376436924552, "2": 0.001930411451950494, "3": 0.0017027720201655324}, "probs_as_returned": {"0": 0.9617730521586384, "1": 0.03459376436924552, "2": 0.001930411451950494, "3": 0.0017027720201655324}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053061192971654236, "usage": {"input_tokens": 228, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "04dae6520bb8e8101bae8f680493c93d7968a872aa767c8efa6e771f7dd62207", "runtime": null} +{"task_id": "original-ordinal-02-0", "family": "ordinal", "split": "public", "group": "original-ordinal-02", "ts": 1790809204.2436585, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "1", "ordinal_ev": 0.9606073357096381, "probs": {"0": 0.06552783210496539, "1": 0.9136649245071803, "2": 0.015479318961105435, "3": 0.005327924426748972}, "probs_as_returned": {"0": 0.06552783210496539, "1": 0.9136649245071803, "2": 0.015479318961105435, "3": 0.005327924426748972}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05437468702439219, "usage": {"input_tokens": 230, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "0e6e436431a630919fea3ae93216e420666bf6163040befd8a94e0180bd9bfc8", "runtime": null} +{"task_id": "original-ordinal-02-1", "family": "ordinal", "split": "public", "group": "original-ordinal-02", "ts": 1790809204.3070943, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "1", "ordinal_ev": 0.9734045635133284, "probs": {"0": 0.03881697376711625, "1": 0.9519256866551472, "2": 0.006293141875028083, "3": 0.0029641977027083286}, "probs_as_returned": {"0": 0.03881697376711625, "1": 0.9519256866551472, "2": 0.006293141875028083, "3": 0.0029641977027083286}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05394969100598246, "usage": {"input_tokens": 231, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "df3421d757e8cfbf75ccfdc1cd1e8675410545a9fc0480a455dab90a1cdfad4c", "runtime": null} +{"task_id": "original-ordinal-03-0", "family": "ordinal", "split": "public", "group": "original-ordinal-03", "ts": 1790809204.3704112, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.9918667458343595, "probs": {"0": 0.0060159904313022965, "1": 0.01058101655212649, "2": 0.9689232497674803, "3": 0.014479743249090765}, "probs_as_returned": {"0": 0.0060159904313022965, "1": 0.01058101655212649, "2": 0.9689232497674803, "3": 0.014479743249090765}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0524212250020355, "usage": {"input_tokens": 227, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c64becc37563b9b6380fe506a259f2628394a45c744ba6e95592c7581078a6a5", "runtime": null} +{"task_id": "original-ordinal-03-1", "family": "ordinal", "split": "public", "group": "original-ordinal-03", "ts": 1790809204.433142, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 2.00299542852447, "probs": {"0": 0.006773356163561288, "1": 0.007211914883152434, "2": 0.9622606732185414, "3": 0.023754055734744896}, "probs_as_returned": {"0": 0.006773356163561288, "1": 0.007211914883152434, "2": 0.9622606732185414, "3": 0.023754055734744896}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05338490998838097, "usage": {"input_tokens": 230, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8acc36887c4bc30b6dce7d21e7bdaf33c374f3bb555c7a2d75b389c68a6aaac1", "runtime": null} +{"task_id": "original-ordinal-04-0", "family": "ordinal", "split": "public", "group": "original-ordinal-04", "ts": 1790809204.4969494, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "3", "ordinal_ev": 2.9704331614645687, "probs": {"0": 0.004185713097107498, "1": 0.0034676031589811955, "2": 0.010074492926146724, "3": 0.9822721908177647}, "probs_as_returned": {"0": 0.004185713097107498, "1": 0.0034676031589811955, "2": 0.010074492926146724, "3": 0.9822721908177647}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05294463399332017, "usage": {"input_tokens": 230, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "46518da9b43216c0dc834411b4f107916f0fb8a6b713574752ec5b914a20986a", "runtime": null} +{"task_id": "original-ordinal-04-1", "family": "ordinal", "split": "public", "group": "original-ordinal-04", "ts": 1790809204.562875, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "3", "ordinal_ev": 2.967596710904972, "probs": {"0": 0.003917633194691461, "1": 0.003455654971049581, "2": 0.013739079568854744, "3": 0.9788876322654043}, "probs_as_returned": {"0": 0.003917633194691461, "1": 0.003455654971049581, "2": 0.013739079568854744, "3": 0.9788876322654043}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05522195098455995, "usage": {"input_tokens": 229, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c28968208f6782b4ecfb6114feccaa455d85d1a6784750c7c2f1e00b22dd2ad4", "runtime": null} +{"task_id": "original-ordinal-05-0", "family": "ordinal", "split": "public", "group": "original-ordinal-05", "ts": 1790809204.6266992, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.2291006823712572, "probs": {"0": 0.8500853530913438, "1": 0.08343242245188087, "2": 0.053778413450949664, "3": 0.012703811005825663}, "probs_as_returned": {"0": 0.8500853530913438, "1": 0.08343242245188087, "2": 0.053778413450949664, "3": 0.012703811005825663}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053230335004627705, "usage": {"input_tokens": 232, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "87cb9bad24ade67598f9ae72932f6e11f3e877b5f6b09707cdc3fb333686c7a5", "runtime": null} +{"task_id": "original-ordinal-05-1", "family": "ordinal", "split": "public", "group": "original-ordinal-05", "ts": 1790809204.6905346, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "0", "ordinal_ev": 0.12180446059244274, "probs": {"0": 0.9151321240993054, "1": 0.054372909340145624, "2": 0.024053348429349676, "3": 0.006441618131199253}, "probs_as_returned": {"0": 0.9151321240993054, "1": 0.054372909340145624, "2": 0.024053348429349676, "3": 0.006441618131199253}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05366079998202622, "usage": {"input_tokens": 232, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a0d7bceae8809d323965af02302ad3a8a5cbbedfee5fa8b7c31e0a71a8927e11", "runtime": null} +{"task_id": "original-ordinal-06-0", "family": "ordinal", "split": "public", "group": "original-ordinal-06", "ts": 1790809204.757591, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.9749699592964916, "probs": {"0": 0.006417617571413263, "1": 0.017511402806940014, "2": 0.9707543823753891, "3": 0.005316597246257736}, "probs_as_returned": {"0": 0.006417617571413263, "1": 0.017511402806940014, "2": 0.9707543823753891, "3": 0.005316597246257736}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05464016000041738, "usage": {"input_tokens": 233, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "6313a7c7ff0ac40a2daf5bcd9c9dfba7c41f36ec9c88265d1ab37c07d5b6cf6b", "runtime": null} +{"task_id": "original-ordinal-06-1", "family": "ordinal", "split": "public", "group": "original-ordinal-06", "ts": 1790809204.8219812, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.9972653111993102, "probs": {"0": 0.002390359574020004, "1": 0.004476413325753633, "2": 0.9866107834271227, "3": 0.006522443673103714}, "probs_as_returned": {"0": 0.002390359574020004, "1": 0.004476413325753633, "2": 0.9866107834271227, "3": 0.006522443673103714}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05333925597369671, "usage": {"input_tokens": 240, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "677d965d8ed4e365b37ba67d9ab5bb542dbe6ce79b302195dd81d7bf1e62e651", "runtime": null} +{"task_id": "original-extraction-01-0", "family": "extraction", "split": "public", "group": "original-extraction-01", "ts": 1790809204.8844278, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "pickup", "ordinal_ev": null, "probs": {"courier": 0.038581401175833197, "pickup": 0.9461486456144061, "post": 0.003556342540584308, "unknown": 0.011713610669176297}, "probs_as_returned": {"courier": 0.038581401175833197, "pickup": 0.9461486456144061, "post": 0.003556342540584308, "unknown": 0.011713610669176297}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05289058299968019, "usage": {"input_tokens": 204, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ef653024d432d805a834601e77391f3cb58a1265a687adfc0290cf3e4dd48c64", "runtime": null} +{"task_id": "original-extraction-01-1", "family": "extraction", "split": "public", "group": "original-extraction-01", "ts": 1790809204.946392, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "pickup", "ordinal_ev": null, "probs": {"courier": 0.02334986205840487, "pickup": 0.9458870455791862, "post": 0.004291643013286111, "unknown": 0.026471449349122916}, "probs_as_returned": {"courier": 0.02334986205840487, "pickup": 0.9458870455791862, "post": 0.004291643013286111, "unknown": 0.026471449349122916}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05214221199275926, "usage": {"input_tokens": 206, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "65341f2040161d63d350f52c0b30ec18f44ba18e49a0aba7c0e2e9920707dabc", "runtime": null} +{"task_id": "original-extraction-02-0", "family": "extraction", "split": "public", "group": "original-extraction-02", "ts": 1790809205.0103025, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.01060657179917826, "pickup": 0.006836726175654022, "post": 0.011293323016761128, "unknown": 0.9712633790084065}, "probs_as_returned": {"courier": 0.01060657179917826, "pickup": 0.006836726175654022, "post": 0.011293323016761128, "unknown": 0.9712633790084065}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053377771051600575, "usage": {"input_tokens": 207, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2cbc6dbaa91beaf5f118ecdcb613a155e8c59ce0ee289866eb7cfe549dc3a1f0", "runtime": null} +{"task_id": "original-extraction-02-1", "family": "extraction", "split": "public", "group": "original-extraction-02", "ts": 1790809205.0758781, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.006844176014044733, "pickup": 0.012037641429207909, "post": 0.008796460047921562, "unknown": 0.9723217225088259}, "probs_as_returned": {"courier": 0.006844176014044733, "pickup": 0.012037641429207909, "post": 0.008796460047921562, "unknown": 0.9723217225088259}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05471387703437358, "usage": {"input_tokens": 207, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "706402a283fd1c116ba3d7d8bac7c4ecd1dd16b91ad58aed76d387e55bd95777", "runtime": null} +{"task_id": "original-extraction-03-0", "family": "extraction", "split": "public", "group": "original-extraction-03", "ts": 1790809205.137541, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9326330187285887, "pickup": 0.03354563366553508, "post": 0.00423150739028277, "unknown": 0.02958984021559345}, "probs_as_returned": {"courier": 0.9326330187285887, "pickup": 0.03354563366553508, "post": 0.00423150739028277, "unknown": 0.02958984021559345}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05275241896742955, "usage": {"input_tokens": 211, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f750849dc9796dc9e4d8fc770549edc1585af464c1d2c0b425f38669c7923807", "runtime": null} +{"task_id": "original-extraction-03-1", "family": "extraction", "split": "public", "group": "original-extraction-03", "ts": 1790809205.1986434, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9787207813254738, "pickup": 0.012116863344787614, "post": 0.0034550659828031225, "unknown": 0.005707289346935534}, "probs_as_returned": {"courier": 0.9787207813254738, "pickup": 0.012116863344787614, "post": 0.0034550659828031225, "unknown": 0.005707289346935534}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05267403495963663, "usage": {"input_tokens": 210, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a40598e7cedac4e233d7400a68103b40d9b5c78068761b890cfba8465e779055", "runtime": null} +{"task_id": "original-extraction-04-0", "family": "extraction", "split": "public", "group": "original-extraction-04", "ts": 1790809205.260276, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "post", "ordinal_ev": null, "probs": {"courier": 0.012648863682625869, "pickup": 0.010478797335157587, "post": 0.9595628201991785, "unknown": 0.017309518783038134}, "probs_as_returned": {"courier": 0.012648863682625869, "pickup": 0.010478797335157587, "post": 0.9595628201991785, "unknown": 0.017309518783038134}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05254928203066811, "usage": {"input_tokens": 208, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "61e8952299c1050ebb6de7d112a4cb515fc78447049a1030ae423e0197d44002", "runtime": null} +{"task_id": "original-extraction-04-1", "family": "extraction", "split": "public", "group": "original-extraction-04", "ts": 1790809205.322474, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "post", "ordinal_ev": null, "probs": {"courier": 0.03782691951412233, "pickup": 0.013019885086963941, "post": 0.9276461528936708, "unknown": 0.021507042505243032}, "probs_as_returned": {"courier": 0.03782691951412233, "pickup": 0.013019885086963941, "post": 0.9276461528936708, "unknown": 0.021507042505243032}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053126003011129797, "usage": {"input_tokens": 204, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1c5fb50f2a22799ee3963752250c55d28d468621bed069456208a1bf66cc32aa", "runtime": null} +{"task_id": "original-extraction-05-0", "family": "extraction", "split": "public", "group": "original-extraction-05", "ts": 1790809205.3848066, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.013695135711406525, "pickup": 0.007313064273085332, "post": 0.003235134664139037, "unknown": 0.9757566653513692}, "probs_as_returned": {"courier": 0.013695135711406525, "pickup": 0.007313064273085332, "post": 0.003235134664139037, "unknown": 0.9757566653513692}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05263980495510623, "usage": {"input_tokens": 207, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "681d434643a44708e09b466737f9258ecf9c8b609398005594ae2d1053ae8489", "runtime": null} +{"task_id": "original-extraction-05-1", "family": "extraction", "split": "public", "group": "original-extraction-05", "ts": 1790809205.4502463, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "unknown", "ordinal_ev": null, "probs": {"courier": 0.01138842444688154, "pickup": 0.005711497827869614, "post": 0.003457613540319971, "unknown": 0.9794424641849289}, "probs_as_returned": {"courier": 0.01138842444688154, "pickup": 0.005711497827869614, "post": 0.003457613540319971, "unknown": 0.9794424641849289}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05324170598760247, "usage": {"input_tokens": 204, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "938cdd36815af27d4bd77103ccc09aca4c0dedb1c07b548d293ff2cb4120d876", "runtime": null} +{"task_id": "original-extraction-06-0", "family": "extraction", "split": "public", "group": "original-extraction-06", "ts": 1790809205.5154507, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9860425408815783, "pickup": 0.003946267989335649, "post": 0.006940756955696869, "unknown": 0.003070434173389162}, "probs_as_returned": {"courier": 0.9860425408815783, "pickup": 0.003946267989335649, "post": 0.006940756955696869, "unknown": 0.003070434173389162}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053146774997003376, "usage": {"input_tokens": 207, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "11fd6ab2bca7aba248333ee05ab6b61bde26051533d6d55af9bad975ccb8ab39", "runtime": null} +{"task_id": "original-extraction-06-1", "family": "extraction", "split": "public", "group": "original-extraction-06", "ts": 1790809205.580963, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "courier", "ordinal_ev": null, "probs": {"courier": 0.9707499602252476, "pickup": 0.012018182452518955, "post": 0.00995631722051821, "unknown": 0.007275540101715229}, "probs_as_returned": {"courier": 0.9707499602252476, "pickup": 0.012018182452518955, "post": 0.00995631722051821, "unknown": 0.007275540101715229}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05304860300384462, "usage": {"input_tokens": 207, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ecfdb5b0c956091c551fd4dbec7ac407e2b75d591ab52e1bbcc4470dcb40284c", "runtime": null} +{"task_id": "original-adequacy-01-0", "family": "adequacy", "split": "public", "group": "original-adequacy-01", "ts": 1790809205.6495514, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.91062577829021, "no": 0.08937422170979004}, "probs_as_returned": {"yes": 0.91062577829021, "no": 0.08937422170979004}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05497507599648088, "usage": {"input_tokens": 202, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8c4b3c4bc2e03dc2032cfeb43262f46bbaad56e3b33c07b0529aa3bf8f82b4ca", "runtime": null} +{"task_id": "original-adequacy-01-1", "family": "adequacy", "split": "public", "group": "original-adequacy-01", "ts": 1790809205.7129061, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7557061813028185, "no": 0.24429381869718148}, "probs_as_returned": {"yes": 0.7557061813028185, "no": 0.24429381869718148}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05479545297566801, "usage": {"input_tokens": 198, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ee410465fc2a9bc60956e30194a3228725becf095f21b28b11f52c17b2e267bb", "runtime": null} +{"task_id": "original-adequacy-02-0", "family": "adequacy", "split": "public", "group": "original-adequacy-02", "ts": 1790809205.775542, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.00791685353903075, "no": 0.9920831464609693}, "probs_as_returned": {"yes": 0.00791685353903075, "no": 0.9920831464609693}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05361347895814106, "usage": {"input_tokens": 199, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e7a7dfc3c0fbea423451f564301ac5af4a799ed1653b8fb94499f10c717666c1", "runtime": null} +{"task_id": "original-adequacy-02-1", "family": "adequacy", "split": "public", "group": "original-adequacy-02", "ts": 1790809205.8362424, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.006989802244620407, "no": 0.9930101977553796}, "probs_as_returned": {"yes": 0.006989802244620407, "no": 0.9930101977553796}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05229574994882569, "usage": {"input_tokens": 199, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "889e8196937e623bdf550d5b5c97a430d74c54da9d8f4dca28263746986ecb22", "runtime": null} +{"task_id": "original-adequacy-03-0", "family": "adequacy", "split": "public", "group": "original-adequacy-03", "ts": 1790809205.9032235, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.08937422170979005, "no": 0.91062577829021}, "probs_as_returned": {"yes": 0.08937422170979005, "no": 0.91062577829021}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053940426965709776, "usage": {"input_tokens": 201, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "4c66dcad35309a5d09be1530c8c148e6ddfa9a96cf9f7e0913a3288476913e5a", "runtime": null} +{"task_id": "original-adequacy-03-1", "family": "adequacy", "split": "public", "group": "original-adequacy-03", "ts": 1790809205.9679885, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.06310763191560703, "no": 0.936892368084393}, "probs_as_returned": {"yes": 0.06310763191560703, "no": 0.936892368084393}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052859511983115226, "usage": {"input_tokens": 205, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d518ce93d9393fc8bd03ea7524bc9390d4bd27f9506efdc24de8045a7d8ae597", "runtime": null} +{"task_id": "original-adequacy-04-0", "family": "adequacy", "split": "public", "group": "original-adequacy-04", "ts": 1790809206.032601, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9471470189565372, "no": 0.05285298104346281}, "probs_as_returned": {"yes": 0.9471470189565372, "no": 0.05285298104346281}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05252470000414178, "usage": {"input_tokens": 205, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "53499cc1b208c2b7327d8ff94ffbe4aef6e01d4fd6b34ebbbefa98d5fc64b77b", "runtime": null} +{"task_id": "original-adequacy-04-1", "family": "adequacy", "split": "public", "group": "original-adequacy-04", "ts": 1790809206.0947678, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8940776460411956, "no": 0.10592235395880445}, "probs_as_returned": {"yes": 0.8940776460411956, "no": 0.10592235395880445}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052899671951308846, "usage": {"input_tokens": 202, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "73e456ab3b73c00d8c7f66aeb3c53ad0bffddfc6ef1bbf52b11072a12a6a41d1", "runtime": null} +{"task_id": "original-adequacy-05-0", "family": "adequacy", "split": "public", "group": "original-adequacy-05", "ts": 1790809206.157961, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.11840617139815589, "no": 0.8815938286018441}, "probs_as_returned": {"yes": 0.11840617139815589, "no": 0.8815938286018441}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05271503503900021, "usage": {"input_tokens": 195, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a1c3bfcbc40c3e524d1b89e5bd82c0739dd7f8f9e5296e7d39b49c6fe34c2a4e", "runtime": null} +{"task_id": "original-adequacy-05-1", "family": "adequacy", "split": "public", "group": "original-adequacy-05", "ts": 1790809206.2213025, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.03267757061976492, "no": 0.967322429380235}, "probs_as_returned": {"yes": 0.03267757061976492, "no": 0.967322429380235}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05269264697562903, "usage": {"input_tokens": 195, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "28fd5ce8d798133a7a154c24dcde5a2866a3098cf110af622430d9c7ec00bafd", "runtime": null} +{"task_id": "original-adequacy-06-0", "family": "adequacy", "split": "public", "group": "original-adequacy-06", "ts": 1790809206.284443, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9583890938912367, "no": 0.04161090610876328}, "probs_as_returned": {"yes": 0.9583890938912367, "no": 0.04161090610876328}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05318323598476127, "usage": {"input_tokens": 211, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "77c7cdcfa4ba8bddae9edafa091ee0fb2417a45640a19dad70b10dba82879c86", "runtime": null} +{"task_id": "original-adequacy-06-1", "family": "adequacy", "split": "public", "group": "original-adequacy-06", "ts": 1790809206.3481486, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5930110454411601, "no": 0.4069889545588399}, "probs_as_returned": {"yes": 0.5930110454411601, "no": 0.4069889545588399}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052703571040183306, "usage": {"input_tokens": 212, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e9275fdaca9d1994c40d4a324d36dee02e1670240d4b278dccac5b15b510e71b", "runtime": null} +{"task_id": "original-routing-01-0", "family": "routing", "split": "public", "group": "original-routing-01", "ts": 1790809206.4116695, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "math", "ordinal_ev": null, "probs": {"coding": 0.030403476101725994, "coding_agent": 0.005588078139470822, "document": 0.0029839776552908095, "general": 0.05693642518396949, "math": 0.9000045761395922, "tools": 0.004083466779950801}, "probs_as_returned": {"coding": 0.030403476101725994, "coding_agent": 0.005588078139470822, "document": 0.0029839776552908095, "general": 0.05693642518396949, "math": 0.9000045761395922, "tools": 0.004083466779950801}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05341902596410364, "usage": {"input_tokens": 248, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "aa6c33e6293a3420a12e045a90fd4bb1f4b4c61f45c3b339a7e1df22dae964c0", "runtime": null} +{"task_id": "original-routing-01-1", "family": "routing", "split": "public", "group": "original-routing-01", "ts": 1790809206.474545, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "math", "ordinal_ev": null, "probs": {"coding": 0.027064128848065867, "coding_agent": 0.006004452348212528, "document": 0.004120910951392922, "general": 0.05068284732390288, "math": 0.9082573436250052, "tools": 0.003870316903420689}, "probs_as_returned": {"coding": 0.027064128848065867, "coding_agent": 0.006004452348212528, "document": 0.004120910951392922, "general": 0.05068284732390288, "math": 0.9082573436250052, "tools": 0.003870316903420689}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052387010015081614, "usage": {"input_tokens": 244, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "02216de0561238249a76e7f8143fd590539aac05685a67b337c0b0de36ea85b3", "runtime": null} +{"task_id": "original-routing-02-0", "family": "routing", "split": "public", "group": "original-routing-02", "ts": 1790809206.5373127, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding", "ordinal_ev": null, "probs": {"coding": 0.9415619813564295, "coding_agent": 0.018084528998433786, "document": 0.01595195139756404, "general": 0.02050220621528298, "math": 0.0024289187126925386, "tools": 0.0014704133195969841}, "probs_as_returned": {"coding": 0.9415619813564295, "coding_agent": 0.018084528998433786, "document": 0.01595195139756404, "general": 0.02050220621528298, "math": 0.0024289187126925386, "tools": 0.0014704133195969841}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05267830297816545, "usage": {"input_tokens": 249, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "30b88b949fb8a490d4ca51a4360305231b105e40338829a45fce5b5da99c3fd1", "runtime": null} +{"task_id": "original-routing-02-1", "family": "routing", "split": "public", "group": "original-routing-02", "ts": 1790809206.6000888, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding", "ordinal_ev": null, "probs": {"coding": 0.973635359123508, "coding_agent": 0.008272706167638342, "document": 0.004703566958129719, "general": 0.00998591082769865, "math": 0.0023589227246364557, "tools": 0.0010435341983889071}, "probs_as_returned": {"coding": 0.973635359123508, "coding_agent": 0.008272706167638342, "document": 0.004703566958129719, "general": 0.00998591082769865, "math": 0.0023589227246364557, "tools": 0.0010435341983889071}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052511359041091055, "usage": {"input_tokens": 243, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "364a4bedda317b54beb55b6720b48ffc580cd03417cb5ce753541f3c9a757586", "runtime": null} +{"task_id": "original-routing-03-0", "family": "routing", "split": "public", "group": "original-routing-03", "ts": 1790809206.662693, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding_agent", "ordinal_ev": null, "probs": {"coding": 0.013544661981601466, "coding_agent": 0.9650356763230314, "document": 0.003406754965701558, "general": 0.00873054421930428, "math": 0.0024894731036984113, "tools": 0.006792889406662867}, "probs_as_returned": {"coding": 0.013544661981601466, "coding_agent": 0.9650356763230314, "document": 0.003406754965701558, "general": 0.00873054421930428, "math": 0.0024894731036984113, "tools": 0.006792889406662867}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05262782197678462, "usage": {"input_tokens": 248, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1ca4132661c691999ae5888963321c54af449a37b2808d879321ebb3ef41076a", "runtime": null} +{"task_id": "original-routing-03-1", "family": "routing", "split": "public", "group": "original-routing-03", "ts": 1790809206.7238944, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coding_agent", "ordinal_ev": null, "probs": {"coding": 0.015370445312191146, "coding_agent": 0.9659803493932506, "document": 0.0038659763929818373, "general": 0.009304926219392514, "math": 0.0028250473602176632, "tools": 0.002653255321966199}, "probs_as_returned": {"coding": 0.015370445312191146, "coding_agent": 0.9659803493932506, "document": 0.0038659763929818373, "general": 0.009304926219392514, "math": 0.0028250473602176632, "tools": 0.002653255321966199}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052455282013397664, "usage": {"input_tokens": 250, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c0365bc6f6978f828d40fed75d6ebef4d3341fe7d4aea696ef00cc3acbeb9a84", "runtime": null} +{"task_id": "original-routing-04-0", "family": "routing", "split": "public", "group": "original-routing-04", "ts": 1790809206.7848377, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "document", "ordinal_ev": null, "probs": {"coding": 0.014210229746939956, "coding_agent": 0.022045907219979437, "document": 0.7877520218832728, "general": 0.14478674264730218, "math": 0.004593678879467563, "tools": 0.02661141962303802}, "probs_as_returned": {"coding": 0.014210229746939956, "coding_agent": 0.022045907219979437, "document": 0.7877520218832728, "general": 0.14478674264730218, "math": 0.004593678879467563, "tools": 0.02661141962303802}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05216074496274814, "usage": {"input_tokens": 244, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "6b1419b71b73d22a69ec7b325f2ef0fb53639da4b608f3edc70cb815f44032dd", "runtime": null} +{"task_id": "original-routing-04-1", "family": "routing", "split": "public", "group": "original-routing-04", "ts": 1790809206.8462627, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "document", "ordinal_ev": null, "probs": {"coding": 0.024413533876144627, "coding_agent": 0.027677322136038972, "document": 0.6787431213908676, "general": 0.24874727547479406, "math": 0.006538077731369333, "tools": 0.013880669390785366}, "probs_as_returned": {"coding": 0.024413533876144627, "coding_agent": 0.027677322136038972, "document": 0.6787431213908676, "general": 0.24874727547479406, "math": 0.006538077731369333, "tools": 0.013880669390785366}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052573898050468415, "usage": {"input_tokens": 243, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8db49b4677bf1aa347acfe7a4698a226ec7235b95f1cb1367e1e1cd8f8f7acdd", "runtime": null} +{"task_id": "original-routing-05-0", "family": "routing", "split": "public", "group": "original-routing-05", "ts": 1790809206.9081194, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "tools", "ordinal_ev": null, "probs": {"coding": 0.013819082412644958, "coding_agent": 0.011448251071317058, "document": 0.021439076042707273, "general": 0.08005466995874373, "math": 0.004756477464671993, "tools": 0.868482443049915}, "probs_as_returned": {"coding": 0.013819082412644958, "coding_agent": 0.011448251071317058, "document": 0.021439076042707273, "general": 0.08005466995874373, "math": 0.004756477464671993, "tools": 0.868482443049915}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05235490301856771, "usage": {"input_tokens": 245, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "23f845e759d97c9113e3c229a7e28eb8bf8925aa5dd69f6f1ed76ba139263e82", "runtime": null} +{"task_id": "original-routing-05-1", "family": "routing", "split": "public", "group": "original-routing-05", "ts": 1790809206.9690306, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "tools", "ordinal_ev": null, "probs": {"coding": 0.01907649307568939, "coding_agent": 0.014842661214610822, "document": 0.013940073466224478, "general": 0.26598655114071845, "math": 0.004506346432653027, "tools": 0.6816478746701038}, "probs_as_returned": {"coding": 0.01907649307568939, "coding_agent": 0.014842661214610822, "document": 0.013940073466224478, "general": 0.26598655114071845, "math": 0.004506346432653027, "tools": 0.6816478746701038}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052585534984245896, "usage": {"input_tokens": 245, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "06d2af9940369adb4aad388ddaf1252704cf65d7feb3195ac348b0cc48bb67ff", "runtime": null} +{"task_id": "original-routing-06-0", "family": "routing", "split": "public", "group": "original-routing-06", "ts": 1790809207.030149, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "general", "ordinal_ev": null, "probs": {"coding": 0.022131200140044424, "coding_agent": 0.009194990510311016, "document": 0.06038820440582165, "general": 0.8965198972789722, "math": 0.0071542561880119735, "tools": 0.0046114514768386536}, "probs_as_returned": {"coding": 0.022131200140044424, "coding_agent": 0.009194990510311016, "document": 0.06038820440582165, "general": 0.8965198972789722, "math": 0.0071542561880119735, "tools": 0.0046114514768386536}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05264645902207121, "usage": {"input_tokens": 243, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1f104f782ee1f0857867e6aca7b44dd1dd1eb12cb9426f124ac370f8adc8b56a", "runtime": null} +{"task_id": "original-routing-06-1", "family": "routing", "split": "public", "group": "original-routing-06", "ts": 1790809207.0928807, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "general", "ordinal_ev": null, "probs": {"coding": 0.02223904433233792, "coding_agent": 0.009239796877608073, "document": 0.05352662420536432, "general": 0.9008885827216597, "math": 0.008150214081207844, "tools": 0.005955737781822105}, "probs_as_returned": {"coding": 0.02223904433233792, "coding_agent": 0.009239796877608073, "document": 0.05352662420536432, "general": 0.9008885827216597, "math": 0.008150214081207844, "tools": 0.005955737781822105}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05244229699019343, "usage": {"input_tokens": 243, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "266f5785402d6d3373eb682286eae27b4e952ea1a49b56738d5b539ae69f0406", "runtime": null} +{"task_id": "hard-opus-a-long_policy-01", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-01", "ts": 1790809207.3091176, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "pay_subject_to_15000_sublimit", "ordinal_ev": null, "probs": {"deny_repeated_seepage": 0.01738839593866782, "deny_vacancy_exclusion": 0.02098937905094514, "pay_full_estimate_less_deductible": 0.006785144709800851, "pay_subject_to_10000_sublimit": 0.1562765432073203, "pay_subject_to_15000_sublimit": 0.7985605370932659}, "probs_as_returned": {"deny_repeated_seepage": 0.01738839593866782, "deny_vacancy_exclusion": 0.02098937905094514, "pay_full_estimate_less_deductible": 0.006785144709800851, "pay_subject_to_10000_sublimit": 0.1562765432073203, "pay_subject_to_15000_sublimit": 0.7985605370932659}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.20261167001444846, "usage": {"input_tokens": 4074, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2f2a82379d53d5754d43b59b72920fad029e3e1a6ebb85a168227ce74302c5db", "runtime": null} +{"task_id": "hard-opus-a-long_policy-04", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-04", "ts": 1790809207.465192, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "vp_and_finance_director", "ordinal_ev": null, "probs": {"budget_holder": 0.02271616887362895, "cfo": 0.08482339827852999, "department_head": 0.08482339827852999, "executive_committee": 0.045294839737184185, "vp_and_finance_director": 0.7623421948321268}, "probs_as_returned": {"budget_holder": 0.02271616887362895, "cfo": 0.08482339827852999, "department_head": 0.08482339827852999, "executive_committee": 0.045294839737184185, "vp_and_finance_director": 0.7623421948321268}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1430607970105484, "usage": {"input_tokens": 2764, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "02232a0f8d1bdb512b59dad9b32b3e42480f6b12cba0ffb20ea8487e32ccb1b8", "runtime": null} +{"task_id": "hard-opus-a-long_policy-08", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-08", "ts": 1790809207.6089377, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "parts_covered_labour_charged", "ordinal_ev": null, "probs": {"full_repair_no_charge": 0.06519889343118179, "not_covered": 0.016398833394742487, "parts_covered_labour_charged": 0.9090784780802361, "pro_rata_replacement_credit": 0.009323795093839524}, "probs_as_returned": {"full_repair_no_charge": 0.06519889343118179, "not_covered": 0.016398833394742487, "parts_covered_labour_charged": 0.9090784780802361, "pro_rata_replacement_credit": 0.009323795093839524}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.12942177499644458, "usage": {"input_tokens": 2524, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7a8d4cd0e656209b7f384d841376140392c98812ec19848c93c96b4186b0f729", "runtime": null} +{"task_id": "hard-opus-a-long_policy-09", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-09", "ts": 1790809207.7590528, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "license_required", "ordinal_ev": null, "probs": {"license_exception_lvs": 0.114786494715961, "license_required": 0.8026723665682973, "no_license_required": 0.047691072005070445, "shipment_prohibited": 0.034850066710671174}, "probs_as_returned": {"license_exception_lvs": 0.114786494715961, "license_required": 0.8026723665682973, "no_license_required": 0.047691072005070445, "shipment_prohibited": 0.034850066710671174}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13513322797371075, "usage": {"input_tokens": 2662, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9f70ddc07927061246ac52b7c22f43a3f8c7ac3d74b33922cfcd8a148228dc4a", "runtime": null} +{"task_id": "hard-opus-a-long_policy-11", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-11", "ts": 1790809207.8993828, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5930110454411601, "no": 0.4069889545588399}, "probs_as_returned": {"yes": 0.5930110454411601, "no": 0.4069889545588399}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.12425446003908291, "usage": {"input_tokens": 2470, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "82532fa79fa3e3177ab930353da968be68f00906348d840a354791986dfe454f", "runtime": null} +{"task_id": "hard-opus-a-long_policy-13", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-13", "ts": 1790809208.044106, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7064759712765732, "no": 0.29352402872342676}, "probs_as_returned": {"yes": 0.7064759712765732, "no": 0.29352402872342676}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1275964709930122, "usage": {"input_tokens": 2559, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1006b0d458f8a939de5fc627fbaf00cedf4edc09e854ada93986546e5025eab7", "runtime": null} +{"task_id": "hard-opus-a-long_policy-17", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-17", "ts": 1790809208.1934156, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "database_oncall", "ordinal_ev": null, "probs": {"database_oncall": 0.8353888582269757, "no_page_ticket_only": 0.07232152988407799, "platform_oncall": 0.03861896981172954, "security_incident_response": 0.04963493453981136, "vendor_support": 0.004035707537405349}, "probs_as_returned": {"database_oncall": 0.8353888582269757, "no_page_ticket_only": 0.07232152988407799, "platform_oncall": 0.03861896981172954, "security_incident_response": 0.04963493453981136, "vendor_support": 0.004035707537405349}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13171831902582198, "usage": {"input_tokens": 2494, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "b7304db345d80c02793e8c9e5e8cecd68378fc01f7d5f3f04f3f5c38e3424dd2", "runtime": null} +{"task_id": "hard-opus-a-long_policy-19", "family": "long_policy", "split": "public", "group": "hard-opus-a-long_policy-19", "ts": 1790809208.3320544, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.37709365395745764, "no": 0.6229063460425424}, "probs_as_returned": {"yes": 0.37709365395745764, "no": 0.6229063460425424}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.12299423001240939, "usage": {"input_tokens": 2465, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "cf979343d20bcbb6902c55a7f6dd6d2cb521742080b6d5f25d6474e994292ac8", "runtime": null} +{"task_id": "hard-opus-a-probability-03", "family": "probability", "split": "public", "group": "hard-opus-a-probability-03", "ts": 1790809208.4016805, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.5777854893190272, "no": 0.4222145106809728}, "probs_as_returned": {"yes": 0.5777854893190272, "no": 0.4222145106809728}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.058104092022404075, "usage": {"input_tokens": 544, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9798d90165dbb8b1d4a77b29b921e234315c147b0af2e2733504b50fa8e3f0a1", "runtime": null} +{"task_id": "hard-opus-a-probability-04", "family": "probability", "split": "public", "group": "hard-opus-a-probability-04", "ts": 1790809208.4892612, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "late", "ordinal_ev": null, "probs": {"early": 0.13202257175482904, "late": 0.7183050733055719, "on_time": 0.149672354939599}, "probs_as_returned": {"early": 0.13202257175482904, "late": 0.7183050733055719, "on_time": 0.149672354939599}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.07457850297214463, "usage": {"input_tokens": 1215, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "de9b5dd11bf4cf2be1d14ba6239cc948eed8317ab5303699fc1b613805c5d3d7", "runtime": null} +{"task_id": "hard-opus-a-probability-07", "family": "probability", "split": "public", "group": "hard-opus-a-probability-07", "ts": 1790809208.5809114, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "escalated", "ordinal_ev": null, "probs": {"escalated": 0.6010911219225122, "reopened": 0.09745097270141673, "resolved_first_contact": 0.30145790537607103}, "probs_as_returned": {"escalated": 0.6010911219225122, "reopened": 0.09745097270141673, "resolved_first_contact": 0.30145790537607103}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.08116815099492669, "usage": {"input_tokens": 1329, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "584616751fcee97f770a90af880d41823d3fe52424a7deb0d813850fa1170c77", "runtime": null} +{"task_id": "hard-opus-a-probability-08", "family": "probability", "split": "public", "group": "hard-opus-a-probability-08", "ts": 1790809208.649015, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.6229063460425425, "no": 0.37709365395745753}, "probs_as_returned": {"yes": 0.6229063460425425, "no": 0.37709365395745753}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.057482282049022615, "usage": {"input_tokens": 504, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ac4c52e290b883931119a265d0859747176e9778c5bdfd173db5d92448456bab", "runtime": null} +{"task_id": "hard-opus-a-temporal_numeric-03", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-03", "ts": 1790809208.7263286, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7318060856600669, "no": 0.2681939143399331}, "probs_as_returned": {"yes": 0.7318060856600669, "no": 0.2681939143399331}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06334954197518528, "usage": {"input_tokens": 600, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9e9ea38b243e69365a4b0db3950f9838ac25af9f77907d1e87fae54bc8f8a856", "runtime": null} +{"task_id": "hard-opus-a-temporal_numeric-06", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-06", "ts": 1790809208.8032174, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.12511350509772579, "no": 0.8748864949022742}, "probs_as_returned": {"yes": 0.12511350509772579, "no": 0.8748864949022742}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06223077198956162, "usage": {"input_tokens": 635, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "87411ef7c717ca1096207d390d8ee3f1971069a9432d91ef17ababbb5fec82d1", "runtime": null} +{"task_id": "hard-opus-a-temporal_numeric-07", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-07", "ts": 1790809208.9138224, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "sep_25", "ordinal_ev": null, "probs": {"sep_24": 0.1443341310190758, "sep_25": 0.3262691984686731, "sep_26": 0.1855050950995479, "sep_27": 0.2384199630201128, "sep_28": 0.10547161239259041}, "probs_as_returned": {"sep_24": 0.1443341310190758, "sep_25": 0.3262691984686731, "sep_26": 0.1855050950995479, "sep_27": 0.2384199630201128, "sep_28": 0.10547161239259041}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.09613108896883205, "usage": {"input_tokens": 1418, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "eb7cf7c782897931be65efbf0917255d8a6a0fe09701155bfc1fb71a4c497251", "runtime": null} +{"task_id": "hard-opus-a-temporal_numeric-09", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-09", "ts": 1790809208.9900064, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.42221451068097277, "no": 0.5777854893190273}, "probs_as_returned": {"yes": 0.42221451068097277, "no": 0.5777854893190273}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06213719199877232, "usage": {"input_tokens": 635, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8b5ee62d8ef60960475c4a5bd70051adaf68aecab73cf4f741c7fa1357cf6118", "runtime": null} +{"task_id": "hard-opus-a-temporal_numeric-12", "family": "temporal_numeric", "split": "public", "group": "hard-opus-a-temporal_numeric-12", "ts": 1790809209.0641646, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "3", "ordinal_ev": 2.102096986624212, "probs": {"0": 0.11983544620580835, "1": 0.10570411896691179, "2": 0.3269884368245393, "3": 0.4474719980027406}, "probs_as_returned": {"0": 0.11983544620580835, "1": 0.10570411896691179, "2": 0.3269884368245393, "3": 0.4474719980027406}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06127866398310289, "usage": {"input_tokens": 708, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "22a1465b4f2232bcfc7ce67579434caa745ebcc4ff7b5f76fb92717f37fa6620", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-02", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-02", "ts": 1790809209.1386704, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "real_incident", "ordinal_ev": null, "probs": {"cannot_determine": 0.02744828681074916, "false_positive": 0.05140225590652218, "real_incident": 0.9211494572827287}, "probs_as_returned": {"cannot_determine": 0.02744828681074916, "false_positive": 0.05140225590652218, "real_incident": 0.9211494572827287}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06232503696810454, "usage": {"input_tokens": 654, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8466f92e056fbef962cc9fa306ee111a36f4e01af8d6fee315b0123410219edc", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-03", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-03", "ts": 1790809209.214224, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "within_limitation", "ordinal_ev": null, "probs": {"insufficient_information": 0.04935003802520305, "time_barred": 0.30439788588738664, "within_limitation": 0.6462520760874103}, "probs_as_returned": {"insufficient_information": 0.04935003802520305, "time_barred": 0.30439788588738664, "within_limitation": 0.6462520760874103}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06194691703421995, "usage": {"input_tokens": 606, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "3d76e0811ac616971cb42ba7b74185856fced76ef002835b82208454ee140f1f", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-07", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-07", "ts": 1790809209.2922268, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "within_policy", "ordinal_ev": null, "probs": {"insufficient_information": 0.04206359082164066, "over_limit": 0.33346268046304034, "within_policy": 0.624473728715319}, "probs_as_returned": {"insufficient_information": 0.04206359082164066, "over_limit": 0.33346268046304034, "within_policy": 0.624473728715319}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06251689500641078, "usage": {"input_tokens": 608, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9188fb2940f548e05de35cc99dbac08dace9431254f47990ea0fd56a203effb1", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-09", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-09", "ts": 1790809209.3716629, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "legitimate", "ordinal_ev": null, "probs": {"compromised": 0.052277255306895254, "inconclusive": 0.010892947754565218, "legitimate": 0.9368297969385395}, "probs_as_returned": {"compromised": 0.052277255306895254, "inconclusive": 0.010892947754565218, "legitimate": 0.9368297969385395}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.061755176982842386, "usage": {"input_tokens": 747, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "66f02cc94bb267fb78cd81e189efb371a585410723356ea5e2cec02c51cfb2a8", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-10", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-10", "ts": 1790809209.4483445, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "send_to_alex_moreno", "ordinal_ev": null, "probs": {"ask_user_which_alex": 0.21763025953150913, "send_to_alex_moreno": 0.6732242882994754, "send_to_alexandra_lind": 0.10914545216901542}, "probs_as_returned": {"ask_user_which_alex": 0.21763025953150913, "send_to_alex_moreno": 0.6732242882994754, "send_to_alexandra_lind": 0.10914545216901542}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06313779199263081, "usage": {"input_tokens": 666, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "34a8239a2cb113abbce415811c60bc1b915e334fb782d2968a5248fa40d478e3", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-11", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-11", "ts": 1790809209.519351, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cannot_determine", "ordinal_ev": null, "probs": {"cannot_determine": 0.9499639519924629, "grade_correct": 0.02345050282102886, "grade_should_be_higher": 0.026585545186508176}, "probs_as_returned": {"cannot_determine": 0.9499639519924629, "grade_correct": 0.02345050282102886, "grade_should_be_higher": 0.026585545186508176}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05861352902138606, "usage": {"input_tokens": 574, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "96737e757a80b85f2f31af57e9d7418e5d56d7d3df7b89f1797c3c84a1a1c1fb", "runtime": null} +{"task_id": "hard-opus-b-ambiguous-13", "family": "ambiguous", "split": "public", "group": "hard-opus-b-ambiguous-13", "ts": 1790809209.5934188, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "cannot_determine", "ordinal_ev": null, "probs": {"cannot_determine": 0.9775256664330146, "charge_data_team": 0.01371996433710697, "charge_platform_team": 0.0026849668364085587, "split_50_50": 0.006069402393469905}, "probs_as_returned": {"cannot_determine": 0.9775256664330146, "charge_data_team": 0.01371996433710697, "charge_platform_team": 0.0026849668364085587, "split_50_50": 0.006069402393469905}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06123842304805294, "usage": {"input_tokens": 717, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9003b08211cb432ab97d6a82667487845e5f352c4cce835f985e6a0f19fa06bb", "runtime": null} +{"task_id": "hard-opus-b-multi_hop-03", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-03", "ts": 1790809209.767027, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "tier3_cfo", "ordinal_ev": null, "probs": {"tier1_team_lead": 0.0497521079639966, "tier2_department_head": 0.17447988465360076, "tier3_cfo": 0.6118982972031438, "tier4_board": 0.16386971017925878}, "probs_as_returned": {"tier1_team_lead": 0.0497521079639966, "tier2_department_head": 0.17447988465360076, "tier3_cfo": 0.6118982972031438, "tier4_board": 0.16386971017925878}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.15930589998606592, "usage": {"input_tokens": 3091, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8d25d8554a148c6db120f685f3be03c4423f3820a6df0b9a7c7d5cc011ea5153", "runtime": null} +{"task_id": "hard-opus-b-multi_hop-04", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-04", "ts": 1790809209.9065125, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "bjorn", "ordinal_ev": null, "probs": {"ana": 0.10338450001046608, "bjorn": 0.5624917858876448, "chen": 0.05520628309044809, "dmitri": 0.14147799705200356, "esra": 0.040341777170236494, "no_page_ticket_only": 0.09709765678920096}, "probs_as_returned": {"ana": 0.10338450001046608, "bjorn": 0.5624917858876448, "chen": 0.05520628309044809, "dmitri": 0.14147799705200356, "esra": 0.040341777170236494, "no_page_ticket_only": 0.09709765678920096}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.12720306095434353, "usage": {"input_tokens": 2490, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7f120f0494faa483542e26e615c0c2022de3f58e3984cc444134ee2deb4eb85f", "runtime": null} +{"task_id": "hard-opus-b-multi_hop-05", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-05", "ts": 1790809210.0580432, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 2.3268150682115487, "probs": {"0": 0.02658575966524453, "1": 0.12758992856431356, "2": 0.3946907024781775, "3": 0.3946907024781775, "4": 0.056442906814086966}, "probs_as_returned": {"0": 0.02658575966524453, "1": 0.12758992856431356, "2": 0.3946907024781775, "3": 0.3946907024781775, "4": 0.056442906814086966}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13762267999118194, "usage": {"input_tokens": 2721, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "b87600cb4eba3bdfbe5d23cb1217e1ec053df427470a7d3c84c88fdab76f1bec", "runtime": null} +{"task_id": "hard-opus-b-multi_hop-07", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-07", "ts": 1790809210.2072744, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "credit_20_percent", "ordinal_ev": null, "probs": {"credit_10_percent": 0.017590746089182055, "credit_20_percent": 0.5207217809221588, "credit_25_percent": 0.12300755553854442, "credit_50_percent": 0.2960639325196957, "credit_full_month": 0.039764112150960935, "no_credit": 0.0028518727794581552}, "probs_as_returned": {"credit_10_percent": 0.017590746089182055, "credit_20_percent": 0.5207217809221588, "credit_25_percent": 0.12300755553854442, "credit_50_percent": 0.2960639325196957, "credit_full_month": 0.039764112150960935, "no_credit": 0.0028518727794581552}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13865585898747668, "usage": {"input_tokens": 2779, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c52de5534a3503b15de4c62bb2bbc35216c5d17a4cbc79ae43bc9f8836063dad", "runtime": null} +{"task_id": "hard-opus-b-multi_hop-08", "family": "multi_hop", "split": "public", "group": "hard-opus-b-multi_hop-08", "ts": 1790809210.3420515, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.12511350509772579, "no": 0.8748864949022742}, "probs_as_returned": {"yes": 0.12511350509772579, "no": 0.8748864949022742}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.12437878298806027, "usage": {"input_tokens": 2452, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7abe0f295d102f8e2859ca13420513b3e8f1733323ff67ecc6774b05e78333a3", "runtime": null} +{"task_id": "hard-opus-b-probability-01", "family": "probability", "split": "public", "group": "hard-opus-b-probability-01", "ts": 1790809210.4582686, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "merchant_wins", "ordinal_ev": null, "probs": {"customer_wins": 0.14739129143438487, "customer_withdraws": 0.2286644595678006, "merchant_wins": 0.6239442489978145}, "probs_as_returned": {"customer_wins": 0.14739129143438487, "customer_withdraws": 0.2286644595678006, "merchant_wins": 0.6239442489978145}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.10440519102849066, "usage": {"input_tokens": 2032, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "89aca6f33dd80a8f753a9f124e7ab17c1d76fb8bc2150fb92e2f3ad34eb39b8e", "runtime": null} +{"task_id": "hard-opus-b-probability-02", "family": "probability", "split": "public", "group": "hard-opus-b-probability-02", "ts": 1790809210.5310342, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "bad_push", "ordinal_ev": null, "probs": {"bad_push": 0.7518757352522394, "database_hardware": 0.10098383660000465, "upstream_provider": 0.14714042814775588}, "probs_as_returned": {"bad_push": 0.7518757352522394, "database_hardware": 0.10098383660000465, "upstream_provider": 0.14714042814775588}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06195314496289939, "usage": {"input_tokens": 611, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e95665a735e7ed69249fcffe52d4a2a3c7e2bfa428420eb9d1f73b2cd56bad6c", "runtime": null} +{"task_id": "hard-opus-b-probability-03", "family": "probability", "split": "public", "group": "hard-opus-b-probability-03", "ts": 1790809210.6239026, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.6798002499763626, "no": 0.32019975002363743}, "probs_as_returned": {"yes": 0.6798002499763626, "no": 0.32019975002363743}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.08181318000424653, "usage": {"input_tokens": 1327, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "bb152e7d362c9ee1962342b6409600bb510b112a9dc89ace6d8581eaa3f09d49", "runtime": null} +{"task_id": "hard-opus-b-probability-04", "family": "probability", "split": "public", "group": "hard-opus-b-probability-04", "ts": 1790809210.6966062, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "attended", "ordinal_ev": null, "probs": {"attended": 0.8908360355378411, "late_cancellation": 0.07712171561686927, "no_show": 0.03204224884528953}, "probs_as_returned": {"attended": 0.8908360355378411, "late_cancellation": 0.07712171561686927, "no_show": 0.03204224884528953}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06057675200281665, "usage": {"input_tokens": 712, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "501bf1ce246df05940b70645c181bde04aada7d0b784e03e511682691b23b21c", "runtime": null} +{"task_id": "hard-opus-b-probability-06", "family": "probability", "split": "public", "group": "hard-opus-b-probability-06", "ts": 1790809210.7848003, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "upheld", "ordinal_ev": null, "probs": {"modified": 0.13231015935842363, "overturned": 0.23270913097863707, "upheld": 0.6349807096629393}, "probs_as_returned": {"modified": 0.13231015935842363, "overturned": 0.23270913097863707, "upheld": 0.6349807096629393}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.07700275501701981, "usage": {"input_tokens": 1130, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "de57b69db19aa126f80d819d739c3676b6dd9609afa22f58c93cf59b7d63e0b9", "runtime": null} +{"task_id": "hard-opus-b-tradeoff-01", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-01", "ts": 1790809210.8662286, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "escalate_to_security", "ordinal_ev": null, "probs": {"escalate_to_engineering": 0.3134898075110609, "escalate_to_security": 0.6655546819611511, "page_oncall": 0.006021181075378121, "process_refund": 0.009946155239190027, "send_kb_article": 0.004988174213219846}, "probs_as_returned": {"escalate_to_engineering": 0.3134898075110609, "escalate_to_security": 0.6655546819611511, "page_oncall": 0.006021181075378121, "process_refund": 0.009946155239190027, "send_kb_article": 0.004988174213219846}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.07040711201261729, "usage": {"input_tokens": 957, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "5911f8b99de1da68ee0ccc62b9de0eda4b63d6aece417b3a0e41543ef161ed8e", "runtime": null} +{"task_id": "hard-opus-b-tradeoff-03", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-03", "ts": 1790809210.9451773, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "order_a", "ordinal_ev": null, "probs": {"order_a": 0.6129149232266266, "order_b": 0.3484816687969538, "order_c": 0.023473188687705322, "order_d": 0.015130219288714327}, "probs_as_returned": {"order_a": 0.6129149232266266, "order_b": 0.3484816687969538, "order_c": 0.023473188687705322, "order_d": 0.015130219288714327}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06697911297669634, "usage": {"input_tokens": 879, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "17a24367b36df85b41d57d49495d497706a388c1801c898c39e3ebda14b486f9", "runtime": null} +{"task_id": "hard-opus-b-tradeoff-06", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-06", "ts": 1790809211.026029, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.844739173437523, "no": 0.15526082656247697}, "probs_as_returned": {"yes": 0.844739173437523, "no": 0.15526082656247697}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0681514159659855, "usage": {"input_tokens": 905, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "35be7e07e1368a008e0e72e54209e833716a3ebe7d9c14a22270dabe056eb444", "runtime": null} +{"task_id": "hard-opus-b-tradeoff-07", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-07", "ts": 1790809211.1066008, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "p0_fix_24h", "ordinal_ev": null, "probs": {"out_of_scope": 0.008135371574581048, "p0_fix_24h": 0.7449709949864798, "p1_fix_7d": 0.1873751928949244, "p2_fix_30d": 0.05688855468489418, "p3_backlog": 0.0026298858591205228}, "probs_as_returned": {"out_of_scope": 0.008135371574581048, "p0_fix_24h": 0.7449709949864798, "p1_fix_7d": 0.1873751928949244, "p2_fix_30d": 0.05688855468489418, "p3_backlog": 0.0026298858591205228}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06805700302356854, "usage": {"input_tokens": 836, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "b2c77292516c7b6e1404af3e545838fca7663aca195d579eb7281b8215bf60b2", "runtime": null} +{"task_id": "hard-opus-b-tradeoff-08", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-08", "ts": 1790809211.1831172, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8678561574454184, "no": 0.13214384255458156}, "probs_as_returned": {"yes": 0.8678561574454184, "no": 0.13214384255458156}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.061771895037963986, "usage": {"input_tokens": 718, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "668d9d26f89aacdd731c98fe3705ddf9004b9b8c815b131e7c6c6385cf1d930b", "runtime": null} +{"task_id": "hard-opus-b-tradeoff-12", "family": "tradeoff", "split": "public", "group": "hard-opus-b-tradeoff-12", "ts": 1790809211.2583113, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "refund_now", "ordinal_ev": null, "probs": {"deny_refund": 0.0072570034281931855, "escalate_to_fraud_team": 0.004980548547941255, "offer_store_credit": 0.011258595089608286, "refund_now": 0.9682766780586276, "wait_for_inspection": 0.008227174875629762}, "probs_as_returned": {"deny_refund": 0.0072570034281931855, "escalate_to_fraud_team": 0.004980548547941255, "offer_store_credit": 0.011258595089608286, "refund_now": 0.9682766780586276, "wait_for_inspection": 0.008227174875629762}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.061137614015024155, "usage": {"input_tokens": 711, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "40bd35993323bf1970d365ba4eff147508411d5a4c58b3973c3864988dcd74ce", "runtime": null} +{"task_id": "hard-opus-c-long_policy-02", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-02", "ts": 1790809211.4583445, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "erase_but_retain_transaction_records", "ordinal_ev": null, "probs": {"erase_all": 0.05664948434336836, "erase_but_retain_transaction_records": 0.8954689046360063, "erase_marketing_only": 0.00714586913522651, "reject_unverified_request": 0.02353652032748053, "restrict_all_pending_legal_hold": 0.017199221557918292}, "probs_as_returned": {"erase_all": 0.05664948434336836, "erase_but_retain_transaction_records": 0.8954689046360063, "erase_marketing_only": 0.00714586913522651, "reject_unverified_request": 0.02353652032748053, "restrict_all_pending_legal_hold": 0.017199221557918292}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1854746930184774, "usage": {"input_tokens": 3728, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f1d27c34232db5a03fa027db95f8d6742ad896d4311ad812bf35af0dde963bb2", "runtime": null} +{"task_id": "hard-opus-c-long_policy-03", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-03", "ts": 1790809211.654099, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "eur_300", "ordinal_ev": null, "probs": {"eur_0": 0.01383331223422414, "eur_150": 0.2328238201782844, "eur_200": 0.2478986155873501, "eur_300": 0.31861107612832196, "eur_400": 0.17013510609182225, "eur_600": 0.016698069779997195}, "probs_as_returned": {"eur_0": 0.01383331223422414, "eur_150": 0.2328238201782844, "eur_200": 0.2478986155873501, "eur_300": 0.31861107612832196, "eur_400": 0.17013510609182225, "eur_600": 0.016698069779997195}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.18053846998373047, "usage": {"input_tokens": 3638, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "40bde6a10c3d8ceecf5a66e27cfb56c52a85f9a7cf4e6ef7b0d785b564356fe9", "runtime": null} +{"task_id": "hard-opus-c-long_policy-04", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-04", "ts": 1790809211.844159, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9439168356016766, "no": 0.05608316439832339}, "probs_as_returned": {"yes": 0.9439168356016766, "no": 0.05608316439832339}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1734917529975064, "usage": {"input_tokens": 3450, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "055bf60c8ca47cc5762584c31cb0bc218e456f7c1b5ac5c9790ce84a2551de4b", "runtime": null} +{"task_id": "hard-opus-c-long_policy-05", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-05", "ts": 1790809212.0368736, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "2", "ordinal_ev": 1.8009979573655441, "probs": {"0": 0.10684458399102813, "1": 0.22683645296660385, "2": 0.4247953847281636, "3": 0.2415235783142044}, "probs_as_returned": {"0": 0.10684458399102813, "1": 0.22683645296660385, "2": 0.4247953847281636, "3": 0.2415235783142044}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.18122948898235336, "usage": {"input_tokens": 3385, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "ec2c8e2532aed313452791b73511d8d8ca977d2378f9aec5b28657ecf01df3d5", "runtime": null} +{"task_id": "hard-opus-c-long_policy-08", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-08", "ts": 1790809212.2250383, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "2", "ordinal_ev": 1.886612524670372, "probs": {"0": 0.014822349797496677, "1": 0.19410266301948487, "2": 0.6807150998981681, "3": 0.11035988728485027}, "probs_as_returned": {"0": 0.014822349797496677, "1": 0.19410266301948487, "2": 0.6807150998981681, "3": 0.11035988728485027}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.17402467294596136, "usage": {"input_tokens": 3402, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d9ed172496e828d2e3d83f732d0a62e58a2d3b1de05cfbd3a9e682ed43ad72d8", "runtime": null} +{"task_id": "hard-opus-c-long_policy-10", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-10", "ts": 1790809212.4262235, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "credit_minus_restocking_fee", "ordinal_ev": null, "probs": {"credit_minus_restocking_fee": 0.8606962036243352, "full_credit": 0.04236513446101119, "reject_return": 0.07933695250396099, "repair_or_replace_only": 0.017601709410692636}, "probs_as_returned": {"credit_minus_restocking_fee": 0.8606962036243352, "full_credit": 0.04236513446101119, "reject_return": 0.07933695250396099, "repair_or_replace_only": 0.017601709410692636}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.18631754297530279, "usage": {"input_tokens": 3764, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "03808242ea9d3a2e52233b7bd4c74c9e5ca8f1ccae427a01456a0236e6bacab9", "runtime": null} +{"task_id": "hard-opus-c-long_policy-11", "family": "long_policy", "split": "public", "group": "hard-opus-c-long_policy-11", "ts": 1790809212.6192064, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8940776460411956, "no": 0.10592235395880445}, "probs_as_returned": {"yes": 0.8940776460411956, "no": 0.10592235395880445}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.17636661499273032, "usage": {"input_tokens": 3528, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "fcc0805f7597795f5c42eca5250edec23dfd2f17ac17b1b54d5510a1448d1016", "runtime": null} +{"task_id": "hard-opus-c-probability-03", "family": "probability", "split": "public", "group": "hard-opus-c-probability-03", "ts": 1790809212.698155, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "kuznets_technik", "ordinal_ev": null, "probs": {"halvorsen_polymer": 0.10720891270072189, "kuznets_technik": 0.7982245521997403, "ortega_seal_works": 0.09456653509953773}, "probs_as_returned": {"halvorsen_polymer": 0.10720891270072189, "kuznets_technik": 0.7982245521997403, "ortega_seal_works": 0.09456653509953773}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06407230603508651, "usage": {"input_tokens": 668, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1426277070dbc1309c27cca6c350545f79b4e9b622a4621a65854b2d47588536", "runtime": null} +{"task_id": "hard-opus-c-temporal_numeric-02", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-02", "ts": 1790809212.7869716, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.1724380672551571, "no": 0.8275619327448429}, "probs_as_returned": {"yes": 0.1724380672551571, "no": 0.8275619327448429}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0733301579602994, "usage": {"input_tokens": 1053, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "cd2ff6eb8372e71316c5a1f9f56fc81f5db6803475dee79dee217f4a7c9f072f", "runtime": null} +{"task_id": "hard-opus-c-temporal_numeric-03", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-03", "ts": 1790809212.870025, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "eur_617_00", "ordinal_ev": null, "probs": {"eur_617_00": 0.36635352271372007, "eur_688_00": 0.2514317984133709, "eur_692_00": 0.22178227377039253, "eur_693_90": 0.111227754246519, "eur_812_00": 0.04920465085599755}, "probs_as_returned": {"eur_617_00": 0.36635352271372007, "eur_688_00": 0.2514317984133709, "eur_692_00": 0.22178227377039253, "eur_693_90": 0.111227754246519, "eur_812_00": 0.04920465085599755}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06951038399711251, "usage": {"input_tokens": 986, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "94288a5d6a5d9dae06ba1f3855eadb143bf0980d13d9321ce1834abdc18d4196", "runtime": null} +{"task_id": "hard-opus-c-temporal_numeric-04", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-04", "ts": 1790809212.9514709, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "usd_2303_01", "ordinal_ev": null, "probs": {"usd_2222_50": 0.014913710681025665, "usd_2303_01": 0.34349499154165725, "usd_2303_33": 0.3226069482266983, "usd_2335_00": 0.2079442332558462, "usd_2340_00": 0.11104011629477252}, "probs_as_returned": {"usd_2222_50": 0.014913710681025665, "usd_2303_01": 0.34349499154165725, "usd_2303_33": 0.3226069482266983, "usd_2335_00": 0.2079442332558462, "usd_2340_00": 0.11104011629477252}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06669761904049665, "usage": {"input_tokens": 899, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "eb2b67ebc8c1fff64698f187925f45d6a15796d127bf54cfe5a3233803a38331", "runtime": null} +{"task_id": "hard-opus-c-temporal_numeric-06", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-06", "ts": 1790809213.0386157, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "sep_16_taxi", "ordinal_ev": null, "probs": {"none_declined": 0.02623440752459831, "sep_15_office_chair": 0.25104537458089066, "sep_16_taxi": 0.5674913765540806, "sep_17_parking": 0.04614147757294348, "sep_18_book_set": 0.08115433880928592, "sep_21_client_dinner": 0.027933024958201016}, "probs_as_returned": {"none_declined": 0.02623440752459831, "sep_15_office_chair": 0.25104537458089066, "sep_16_taxi": 0.5674913765540806, "sep_17_parking": 0.04614147757294348, "sep_18_book_set": 0.08115433880928592, "sep_21_client_dinner": 0.027933024958201016}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.07447077397955582, "usage": {"input_tokens": 1102, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "a33eb38f3f33daef06f06f861b545950755401a093b3d6e7bea80bc47f9acc8e", "runtime": null} +{"task_id": "hard-opus-c-temporal_numeric-08", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-08", "ts": 1790809213.1246116, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "eur_498_81", "ordinal_ev": null, "probs": {"eur_497_40": 0.22970119120589697, "eur_498_81": 0.37943445912969975, "eur_506_06": 0.13905597950085433, "eur_513_69": 0.19029314129726177, "eur_532_01": 0.06151522886628714}, "probs_as_returned": {"eur_497_40": 0.22970119120589697, "eur_498_81": 0.37943445912969975, "eur_506_06": 0.13905597950085433, "eur_513_69": 0.19029314129726177, "eur_532_01": 0.06151522886628714}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0733960079960525, "usage": {"input_tokens": 1105, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c6071d979d268109018b771eea3ca8943f18bf68d39ed3beddd9c1113a8e5ebd", "runtime": null} +{"task_id": "hard-opus-c-temporal_numeric-12", "family": "temporal_numeric", "split": "public", "group": "hard-opus-c-temporal_numeric-12", "ts": 1790809213.2045684, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "expired_24_month_term", "ordinal_ev": null, "probs": {"covered": 0.1309008006401481, "expired_24_month_term": 0.488790641617104, "expired_30_month_cap": 0.3803085577427478}, "probs_as_returned": {"covered": 0.1309008006401481, "expired_24_month_term": 0.488790641617104, "expired_30_month_cap": 0.3803085577427478}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06677124102134258, "usage": {"input_tokens": 935, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "56f6200b6e7061ab62085805a0fc8cc9cfafd8698254664c98fa97c8e9a0b4c9", "runtime": null} +{"task_id": "hard-sol-a-adversarial-01", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-01", "ts": 1790809213.2727485, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "deny_missing_approval", "ordinal_ev": null, "probs": {"deny_inactive_user": 0.013568707364640402, "deny_missing_approval": 0.8527472004718343, "grant_access": 0.03265819804374573, "grant_owner_exception": 0.10102589411977958}, "probs_as_returned": {"deny_inactive_user": 0.013568707364640402, "deny_missing_approval": 0.8527472004718343, "grant_access": 0.03265819804374573, "grant_owner_exception": 0.10102589411977958}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05631913000252098, "usage": {"input_tokens": 390, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "fe3ed457e399ae5e550c5a500983b8d508aace8769b68bdaef8f0726857a3c6b", "runtime": null} +{"task_id": "hard-sol-a-adversarial-06", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-06", "ts": 1790809213.3412693, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.016659759016074603, "no": 0.9833402409839254}, "probs_as_returned": {"yes": 0.016659759016074603, "no": 0.9833402409839254}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05567461298778653, "usage": {"input_tokens": 350, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "57a011077cce0e430469b042d551f53571a49358b4f8ade1a6638a4f3c6daef1", "runtime": null} +{"task_id": "hard-sol-a-adversarial-07", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-07", "ts": 1790809213.4099622, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "retain_until_2031", "ordinal_ev": null, "probs": {"delete_now": 0.010441518332080891, "needs_signature_review": 0.004918167733746614, "retain_indefinitely": 0.02849120141460286, "retain_until_2031": 0.9561491125195697}, "probs_as_returned": {"delete_now": 0.010441518332080891, "needs_signature_review": 0.004918167733746614, "retain_indefinitely": 0.02849120141460286, "retain_until_2031": 0.9561491125195697}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0558955809683539, "usage": {"input_tokens": 406, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "5339d6bc0c47eba8d3c5be8bde4ebc0872f0e4cdcc472d426c995be9e46bd8fb", "runtime": null} +{"task_id": "hard-sol-a-adversarial-08", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-08", "ts": 1790809213.476983, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9502009729447547, "no": 0.049799027055245304}, "probs_as_returned": {"yes": 0.9502009729447547, "no": 0.049799027055245304}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053810442972462624, "usage": {"input_tokens": 327, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9ec3bd77a00e9ebb2cd984b9a7b4a3810f367bfa8126fe2c2f08be1a2cdfb213", "runtime": null} +{"task_id": "hard-sol-a-adversarial-09", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-09", "ts": 1790809213.5442324, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "2", "ordinal_ev": 1.8962367066579777, "probs": {"0": 0.012249937955791055, "1": 0.09711242419678795, "2": 0.8727886310810733, "3": 0.01784900676634773}, "probs_as_returned": {"0": 0.012249937955791055, "1": 0.09711242419678795, "2": 0.8727886310810733, "3": 0.01784900676634773}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05440572102088481, "usage": {"input_tokens": 344, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7da5be6197a511977330c5970af3d26269046e3766fc6a8be950dfbffd047478", "runtime": null} +{"task_id": "hard-sol-a-adversarial-11", "family": "adversarial", "split": "public", "group": "hard-sol-a-adversarial-11", "ts": 1790809213.61126, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "standard_research", "ordinal_ev": null, "probs": {"ethics_review": 0.023754425327906448, "major_grants": 0.009869402226303762, "manual_hold": 0.004100502827029603, "standard_research": 0.9622756696187603}, "probs_as_returned": {"ethics_review": 0.023754425327906448, "major_grants": 0.009869402226303762, "manual_hold": 0.004100502827029603, "standard_research": 0.9622756696187603}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05389246204867959, "usage": {"input_tokens": 411, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "3aaf464d92f7813ffeab555b8f78e471e959e4bdbdc87472bbba06a801f70a25", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-01", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-01", "ts": 1790809213.7695267, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "accept_s2_dusk", "ordinal_ev": null, "probs": {"accept_s2_dusk": 0.8820299897169496, "reject_cutoff": 0.04077512878367174, "reject_tier": 0.033779658694117705, "reroute_s3": 0.04341522280526092}, "probs_as_returned": {"accept_s2_dusk": 0.8820299897169496, "reject_cutoff": 0.04077512878367174, "reject_tier": 0.033779658694117705, "reroute_s3": 0.04341522280526092}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.14446308702463284, "usage": {"input_tokens": 2819, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "6fc9200768352de8056323df13a56b3dadc73fe0c39cc30b8df3019e72aed7dd", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-02", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-02", "ts": 1790809213.9349668, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "security_primary", "ordinal_ev": null, "probs": {"executive_bridge": 0.019116772677897714, "heron_q2": 0.08092609404342527, "insufficient_information": 0.12554962605057535, "security_primary": 0.7744075072281016}, "probs_as_returned": {"executive_bridge": 0.019116772677897714, "heron_q2": 0.08092609404342527, "insufficient_information": 0.12554962605057535, "security_primary": 0.7744075072281016}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.15237322601024061, "usage": {"input_tokens": 2867, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1933e847a59ad105b9a0412a3abd2caffb59d1a0fff6269a8c78f31a4a78a1c1", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-05", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-05", "ts": 1790809214.0929408, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "require_director", "ordinal_ev": null, "probs": {"pay_manager_approved": 0.2549478953776852, "reject_category": 0.034241851845298925, "require_director": 0.6956624298246201, "require_finance_board": 0.015147822952395761}, "probs_as_returned": {"pay_manager_approved": 0.2549478953776852, "reject_category": 0.034241851845298925, "require_director": 0.6956624298246201, "require_finance_board": 0.015147822952395761}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13955179101321846, "usage": {"input_tokens": 2779, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "99fc0ad2c1822856a2a6f2289444129170b8141738e05a82d59caa16de837e6e", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-07", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-07", "ts": 1790809214.2502005, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "admit", "ordinal_ev": null, "probs": {"admit": 0.8016965785406263, "deny_prerequisite": 0.11464694915891029, "deny_score": 0.061220315486799355, "manual_equivalency": 0.02243615681366412}, "probs_as_returned": {"admit": 0.8016965785406263, "deny_prerequisite": 0.11464694915891029, "deny_score": 0.061220315486799355, "manual_equivalency": 0.02243615681366412}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.14079474698519334, "usage": {"input_tokens": 2760, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f7f425300c11113bc2dc3188beef9366b119b5eee1128f6031b6eab91057d46a", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-08", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-08", "ts": 1790809214.4081686, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "local_maintenance", "ordinal_ev": null, "probs": {"central_engineering": 0.2039312595652488, "insufficient_log": 0.025724259265304274, "local_maintenance": 0.7614902860659044, "safety_shutdown": 0.00885419510354259}, "probs_as_returned": {"central_engineering": 0.2039312595652488, "insufficient_log": 0.025724259265304274, "local_maintenance": 0.7614902860659044, "safety_shutdown": 0.00885419510354259}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1405503709684126, "usage": {"input_tokens": 2730, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8c2d0c17814f1d0cf12b872bf78ce9d0e30db9a214d8f832107748217101f541", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-09", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-09", "ts": 1790809214.5639656, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "require_steward", "ordinal_ev": null, "probs": {"confirm_l3": 0.06689419587938847, "reject_capacity": 0.03803366795607197, "require_steward": 0.8759975739130693, "switch_l4": 0.019074562251470206}, "probs_as_returned": {"confirm_l3": 0.06689419587938847, "reject_capacity": 0.03803366795607197, "require_steward": 0.8759975739130693, "switch_l4": 0.019074562251470206}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13962874899152666, "usage": {"input_tokens": 2744, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "01638a8d1dc1846b5fa45ec7ac194c86eec8df1977a905fd3bf9632b0a8db308", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-10", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-10", "ts": 1790809214.726507, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "reduce_to_30_days", "ordinal_ev": null, "probs": {"approve_45_days": 0.2266260914546683, "deny_sensitive": 0.013465071955275236, "needs_field_review": 0.013465071955275236, "reduce_to_30_days": 0.7464437646347811}, "probs_as_returned": {"approve_45_days": 0.2266260914546683, "deny_sensitive": 0.013465071955275236, "needs_field_review": 0.013465071955275236, "reduce_to_30_days": 0.7464437646347811}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1475998309906572, "usage": {"input_tokens": 2808, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8e434561785fe80ce2dcb9f4d86817ecf7c82012e506b6ded9575a2468407dd4", "runtime": null} +{"task_id": "hard-sol-a-multi_hop-12", "family": "multi_hop", "split": "public", "group": "hard-sol-a-multi_hop-12", "ts": 1790809214.885143, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "covered_recurrence", "ordinal_ev": null, "probs": {"covered_recurrence": 0.5488178934874424, "expired": 0.01853987381447562, "needs_authentication": 0.4270133837934193, "wrong_issue_class": 0.0056288489046627905}, "probs_as_returned": {"covered_recurrence": 0.5488178934874424, "expired": 0.01853987381447562, "needs_authentication": 0.4270133837934193, "wrong_issue_class": 0.0056288489046627905}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.14252410997869447, "usage": {"input_tokens": 2755, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7911256c1c91328300a6a6ce2974753dad6de31758601891c6e22ad427da64eb", "runtime": null} +{"task_id": "hard-sol-a-trap-02", "family": "trap", "split": "public", "group": "hard-sol-a-trap-02", "ts": 1790809214.9577649, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.03075154512433679, "no": 0.9692484548756632}, "probs_as_returned": {"yes": 0.03075154512433679, "no": 0.9692484548756632}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05614370299736038, "usage": {"input_tokens": 401, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "e427ef9aa71cab8fa79a043d4d39b76cd3aa138c50fba9f89a975c07779f9e7a", "runtime": null} +{"task_id": "hard-sol-a-trap-04", "family": "trap", "split": "public", "group": "hard-sol-a-trap-04", "ts": 1790809215.0261512, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.059498364613396776, "no": 0.9405016353866033}, "probs_as_returned": {"yes": 0.059498364613396776, "no": 0.9405016353866033}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054047968005761504, "usage": {"input_tokens": 307, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "0585363dda67ae44e2513a65dd48560ba6ec7d4cb7c74efdc12bd8e4cb072f44", "runtime": null} +{"task_id": "hard-sol-a-trap-05", "family": "trap", "split": "public", "group": "hard-sol-a-trap-05", "ts": 1790809215.0979404, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "1", "ordinal_ev": 0.8643000081318148, "probs": {"0": 0.34057212036202084, "1": 0.4660607766073019, "2": 0.1818620775675192, "3": 0.01150502546315819}, "probs_as_returned": {"0": 0.34057212036202084, "1": 0.4660607766073019, "2": 0.1818620775675192, "3": 0.01150502546315819}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05733501905342564, "usage": {"input_tokens": 454, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c2d36495402d4e969639ee136895cae37c88246420b6ce93b0466cdbfdb71cec", "runtime": null} +{"task_id": "hard-sol-a-trap-06", "family": "trap", "split": "public", "group": "hard-sol-a-trap-06", "ts": 1790809215.1668472, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "keep_open_unresolved", "ordinal_ev": null, "probs": {"await_internal_metrics": 0.008337830351487158, "close_agent_verified": 0.0025314294996148625, "close_confirmed": 0.007830803388441922, "keep_open_unresolved": 0.981299936760456}, "probs_as_returned": {"await_internal_metrics": 0.008337830351487158, "close_agent_verified": 0.0025314294996148625, "close_confirmed": 0.007830803388441922, "keep_open_unresolved": 0.981299936760456}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.055784745956771076, "usage": {"input_tokens": 459, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f4d25941bacabb117fab4bcb7d9bc7db1f25d7fb749e7667c22c37bed8bd957f", "runtime": null} +{"task_id": "hard-sol-a-trap-08", "family": "trap", "split": "public", "group": "hard-sol-a-trap-08", "ts": 1790809215.2373548, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "late", "ordinal_ev": null, "probs": {"canceled": 0.0052844814699961285, "indeterminate": 0.00282185985242437, "late": 0.9648903851080965, "on_time": 0.027003273569482986}, "probs_as_returned": {"canceled": 0.0052844814699961285, "indeterminate": 0.00282185985242437, "late": 0.9648903851080965, "on_time": 0.027003273569482986}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.055416371033061296, "usage": {"input_tokens": 386, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "32eebd11e8fd33750e7ee0e606d9db250051249d5a4d1d896e1ff126c7dd0479", "runtime": null} +{"task_id": "hard-sol-a-trap-10", "family": "trap", "split": "public", "group": "hard-sol-a-trap-10", "ts": 1790809215.307824, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "blocked_second_review", "ordinal_ev": null, "probs": {"blocked_failed_test": 0.014370727476242724, "blocked_second_review": 0.9616283560317681, "ready_one_review": 0.00869971844133038, "ready_two_reviews": 0.015301198050658662}, "probs_as_returned": {"blocked_failed_test": 0.014370727476242724, "blocked_second_review": 0.9616283560317681, "ready_one_review": 0.00869971844133038, "ready_two_reviews": 0.015301198050658662}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054895692970603704, "usage": {"input_tokens": 312, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "645714095c3517a286448ea05476c1a272db02a3e2956dcba7b9d95fc29bfa77", "runtime": null} +{"task_id": "hard-sol-a-trap-13", "family": "trap", "split": "public", "group": "hard-sol-a-trap-13", "ts": 1790809215.3778539, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.022659190258890113, "no": 0.9773408097411099}, "probs_as_returned": {"yes": 0.022659190258890113, "no": 0.9773408097411099}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05490435898536816, "usage": {"input_tokens": 315, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8c711646527e8dd214ba07fa35a62f9f8e69c4fade7519e007984c39f32c97d9", "runtime": null} +{"task_id": "hard-sol-a-trap-15", "family": "trap", "split": "public", "group": "hard-sol-a-trap-15", "ts": 1790809215.4506633, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.14720923592665844, "no": 0.8527907640733415}, "probs_as_returned": {"yes": 0.14720923592665844, "no": 0.8527907640733415}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0558483189670369, "usage": {"input_tokens": 417, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d6286ba9f9d2d151610c9eeb9ea6f0517c5c3577b1697286868a9a1dd1956d45", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-01", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-01", "ts": 1790809215.520291, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8879876242264044, "no": 0.11201237577359557}, "probs_as_returned": {"yes": 0.8879876242264044, "no": 0.11201237577359557}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05428842600667849, "usage": {"input_tokens": 402, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "624ac2d51406b9496395ec7f53287010799fcbb7ba0e0ad61979a6876eea35e4", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-02", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-02", "ts": 1790809215.5891745, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.2681939143399331, "no": 0.7318060856600669}, "probs_as_returned": {"yes": 0.2681939143399331, "no": 0.7318060856600669}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0539684469695203, "usage": {"input_tokens": 326, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c91de7bcf9a3d9ec2fe4658c95f02a5f1f5d584beab81b6d28572003f5db14a7", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-05", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-05", "ts": 1790809215.657265, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8363317223730603, "no": 0.1636682776269397}, "probs_as_returned": {"yes": 0.8363317223730603, "no": 0.1636682776269397}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05412257899297401, "usage": {"input_tokens": 346, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "f33f6fbb31838b6c5a3f863e7d71d441ae5337cb8012810bd2dc02fab35ced91", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-08", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-08", "ts": 1790809215.724374, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8815938286018441, "no": 0.1184061713981559}, "probs_as_returned": {"yes": 0.8815938286018441, "no": 0.1184061713981559}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053553084027953446, "usage": {"input_tokens": 328, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7f0ac2d8398b9c16d9e892981e7d9b4ef66a0e7149d636ef856cccc4380ca588", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-10", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-10", "ts": 1790809215.79289, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.3340072352673295, "no": 0.6659927647326704}, "probs_as_returned": {"yes": 0.3340072352673295, "no": 0.6659927647326704}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054366590979043394, "usage": {"input_tokens": 347, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "20bb55fad0fa9d3946d9f9e21d60db3671dc86a63de843ca0a5f8f2b1b20ee58", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-14", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-14", "ts": 1790809215.860069, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.011493812770397228, "no": 0.9885061872296028}, "probs_as_returned": {"yes": 0.011493812770397228, "no": 0.9885061872296028}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05352323700208217, "usage": {"input_tokens": 364, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "9b48e46382dc5045194b64eee70e0177aa3b445a59cfc11d5abfa8a8e7a3473c", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-15", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-15", "ts": 1790809215.9248083, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.9405016353866033, "no": 0.05949836461339675}, "probs_as_returned": {"yes": 0.9405016353866033, "no": 0.05949836461339675}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.053095882991328835, "usage": {"input_tokens": 320, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "d318e29eaafb9dff49cafa9ad2a3118e2670e0bd6d4c077b401a88253812c04d", "runtime": null} +{"task_id": "hard-sol-b-judge_hard-18", "family": "judge_hard", "split": "public", "group": "hard-sol-b-judge_hard-18", "ts": 1790809215.989629, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.016659759016074603, "no": 0.9833402409839254}, "probs_as_returned": {"yes": 0.016659759016074603, "no": 0.9833402409839254}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.052814981958363205, "usage": {"input_tokens": 288, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "6c6d6ef03ed96d870914d3f3f7473a28f67635247beb0f44587c0f81c0ebb1ca", "runtime": null} +{"task_id": "hard-sol-b-long_policy-01", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-01", "ts": 1790809216.154472, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "approve_lodging_only", "ordinal_ev": null, "probs": {"approve_all": 0.08909463191094494, "approve_lodging_only": 0.706305265580735, "approve_meals_only": 0.14717197021020476, "reject_entire_claim": 0.05742813229811534}, "probs_as_returned": {"approve_all": 0.08909463191094494, "approve_lodging_only": 0.706305265580735, "approve_meals_only": 0.14717197021020476, "reject_entire_claim": 0.05742813229811534}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.15017816401086748, "usage": {"input_tokens": 2866, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "0a797809dbe02859d6b88d097608977fd466fd2f205bdf3a24d879ede59832fa", "runtime": null} +{"task_id": "hard-sol-b-long_policy-02", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-02", "ts": 1790809216.3194458, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "deny_current_export", "ordinal_ev": null, "probs": {"analytics_owner_only": 0.012159429361438828, "deny_current_export": 0.8663400458018994, "privacy_and_security": 0.10263625163294894, "privacy_only": 0.018864273203712912}, "probs_as_returned": {"analytics_owner_only": 0.012159429361438828, "deny_current_export": 0.8663400458018994, "privacy_and_security": 0.10263625163294894, "privacy_only": 0.018864273203712912}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.14695013902382925, "usage": {"input_tokens": 2716, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "c49d985b607e7e874e1e52c8caa058c13476635c07bf5e8fa880a7a945d17bb8", "runtime": null} +{"task_id": "hard-sol-b-long_policy-05", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-05", "ts": 1790809216.4803543, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "delete_profile_omar_april_and_table", "ordinal_ev": null, "probs": {"delete_everything": 0.021894364677177067, "delete_nothing": 0.0233119742695315, "delete_only_profile": 0.34608827612402576, "delete_profile_omar_april_and_table": 0.6087053849292656}, "probs_as_returned": {"delete_everything": 0.021894364677177067, "delete_nothing": 0.0233119742695315, "delete_only_profile": 0.34608827612402576, "delete_profile_omar_april_and_table": 0.6087053849292656}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.142696833005175, "usage": {"input_tokens": 2677, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "41ae22095f91cdecaf70961c882b021ea2940fd9505fc769460309eac1231196", "runtime": null} +{"task_id": "hard-sol-b-long_policy-06", "family": "long_policy", "split": "public", "group": "hard-sol-b-long_policy-06", "ts": 1790809216.643411, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "east_ward", "ordinal_ev": null, "probs": {"east_ward": 0.3411456641180011, "harbor_neighbors": 0.3009168221241823, "no_award": 0.016791849639815445, "river_arts": 0.3411456641180011}, "probs_as_returned": {"east_ward": 0.3411456641180011, "harbor_neighbors": 0.3009168221241823, "no_award": 0.016791849639815445, "river_arts": 0.3411456641180011}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1443015889963135, "usage": {"input_tokens": 2760, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8b3bcbf6be613edf88153fa8c94ddb40503ac8b9d3a12a37265091407be54517", "runtime": null} +{"task_id": "hard-sol-b-routing_hard-01", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-01", "ts": 1790809216.715403, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "office_file_repair", "ordinal_ev": null, "probs": {"data_pipeline": 0.002432868028110054, "desktop_support": 0.003774378037715521, "finance_review": 0.004278965637903766, "office_file_repair": 0.9430929283602817, "spreadsheet_formula": 0.04642085993598908}, "probs_as_returned": {"data_pipeline": 0.002432868028110054, "desktop_support": 0.003774378037715521, "finance_review": 0.004278965637903766, "office_file_repair": 0.9430929283602817, "spreadsheet_formula": 0.04642085993598908}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05698318500071764, "usage": {"input_tokens": 440, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2caef222c6bcda04c92dd802eb05897d9ed7f65d72794538caa832962b32cace", "runtime": null} +{"task_id": "hard-sol-b-routing_hard-02", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-02", "ts": 1790809216.7841904, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "ledger_reconciliation", "ordinal_ev": null, "probs": {"customer_support": 0.024586117348690935, "fraud_review": 0.0015553761372732856, "ledger_reconciliation": 0.9354018336737445, "mobile_app": 0.03364522339881921, "payment_disputes": 0.004811449441472035}, "probs_as_returned": {"customer_support": 0.024586117348690935, "fraud_review": 0.0015553761372732856, "ledger_reconciliation": 0.9354018336737445, "mobile_app": 0.03364522339881921, "payment_disputes": 0.004811449441472035}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05426047503715381, "usage": {"input_tokens": 423, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "53a03d6389c173161c9a5cab25bfcca6a97602cf58913f6b3e9b2f32d2216f1d", "runtime": null} +{"task_id": "hard-sol-b-routing_hard-03", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-03", "ts": 1790809216.8530664, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "network_dns", "ordinal_ev": null, "probs": {"api_platform": 0.004842894208676956, "authoritative_dns": 0.005156460174491347, "endpoint_devices": 0.04634319529093982, "network_dns": 0.9415150632539939, "tls_certificates": 0.002142387071897831}, "probs_as_returned": {"api_platform": 0.004842894208676956, "authoritative_dns": 0.005156460174491347, "endpoint_devices": 0.04634319529093982, "network_dns": 0.9415150632539939, "tls_certificates": 0.002142387071897831}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05350387701764703, "usage": {"input_tokens": 421, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "4766d3bfbaee0e19f66a18d6d6a8f391b9d9b02bf7cfa4c997876ebe8438985b", "runtime": null} +{"task_id": "hard-sol-b-routing_hard-07", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-07", "ts": 1790809216.9224772, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "release_notarization", "ordinal_ev": null, "probs": {"ci_infrastructure": 0.0026358552072950053, "code_signing": 0.03044686823684439, "endpoint_security": 0.0026358552072950053, "macos_runtime": 0.004635982743818916, "release_notarization": 0.9596454386047467}, "probs_as_returned": {"ci_infrastructure": 0.0026358552072950053, "code_signing": 0.03044686823684439, "endpoint_security": 0.0026358552072950053, "macos_runtime": 0.004635982743818916, "release_notarization": 0.9596454386047467}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.054628726036753505, "usage": {"input_tokens": 404, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "bc156039aa6fc1da3740ac581032e1999fd61b3d35f3d45cb4c4ff866e72ac4c", "runtime": null} +{"task_id": "hard-sol-b-routing_hard-09", "family": "routing_hard", "split": "public", "group": "hard-sol-b-routing_hard-09", "ts": 1790809216.9937506, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "source_access", "ordinal_ev": null, "probs": {"git_client": 0.020297759877090524, "identity_login": 0.006561567668597966, "repository_admin": 0.03352908025315896, "source_access": 0.9321728248733867, "vpn_network": 0.007438767327765931}, "probs_as_returned": {"git_client": 0.020297759877090524, "identity_login": 0.006561567668597966, "repository_admin": 0.03352908025315896, "source_access": 0.9321728248733867, "vpn_network": 0.007438767327765931}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.056705471011810005, "usage": {"input_tokens": 410, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "87ef15f6abf2320d990761e03e9afc099588eccc88aa6f757cf7b00cc33344df", "runtime": null} +{"task_id": "hard-sol-b-temporal_numeric-01", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-01", "ts": 1790809217.0628514, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "late_by_under_2h", "ordinal_ev": null, "probs": {"before_open": 0.07619692419201703, "late_by_2h_or_more": 0.2672214674340832, "late_by_under_2h": 0.38936014093981663, "within_window": 0.2672214674340832}, "probs_as_returned": {"before_open": 0.07619692419201703, "late_by_2h_or_more": 0.2672214674340832, "late_by_under_2h": 0.38936014093981663, "within_window": 0.2672214674340832}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05575839604716748, "usage": {"input_tokens": 402, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "b09f81b927360f54d7430ef9671adba0a643694fc0f12cce9304a68aa08e1e8a", "runtime": null} +{"task_id": "hard-sol-b-temporal_numeric-02", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-02", "ts": 1790809217.1336186, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "under_100gb_below", "ordinal_ev": null, "probs": {"at_least_100gb_below": 0.2777216041744788, "exactly_equal": 0.20294399073523287, "over_limit": 0.1392823299139878, "under_100gb_below": 0.3800520751763005}, "probs_as_returned": {"at_least_100gb_below": 0.2777216041744788, "exactly_equal": 0.20294399073523287, "over_limit": 0.1392823299139878, "under_100gb_below": 0.3800520751763005}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05550732999108732, "usage": {"input_tokens": 415, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "96dcfc32e46af8dc37353d734fcdb7b0de588e34bb1d3ad4aadc1323f2dd9d42", "runtime": null} +{"task_id": "hard-sol-b-temporal_numeric-03", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-03", "ts": 1790809217.200724, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "12_credits", "ordinal_ev": null, "probs": {"12_credits": 0.38418415861441135, "15_credits": 0.11664127258300278, "6_credits": 0.21843347437078164, "9_credits": 0.28074109443180434}, "probs_as_returned": {"12_credits": 0.38418415861441135, "15_credits": 0.11664127258300278, "6_credits": 0.21843347437078164, "9_credits": 0.28074109443180434}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05353608098812401, "usage": {"input_tokens": 354, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2fb1e2db4ddc2893412a0e7ce87069537387ab579082b580342c3d7589912267", "runtime": null} +{"task_id": "hard-sol-b-temporal_numeric-04", "family": "temporal_numeric", "split": "public", "group": "hard-sol-b-temporal_numeric-04", "ts": 1790809217.2653818, "status": "ok", "ok": true, "valid": true, "correct": false, "predicted": "narrow_fail", "ordinal_ev": null, "probs": {"clear_fail": 0.324620580367349, "narrow_fail": 0.6472762711048763, "pass": 0.02810314852777479}, "probs_as_returned": {"clear_fail": 0.324620580367349, "narrow_fail": 0.6472762711048763, "pass": 0.02810314852777479}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05362547701224685, "usage": {"input_tokens": 363, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "376e78cdba2b9e4eb64b0144446014ddefba0ed5951cb4436a5e1073fe67e268", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-03", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-03", "ts": 1790809217.333067, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.03075154512433679, "no": 0.9692484548756632}, "probs_as_returned": {"yes": 0.03075154512433679, "no": 0.9692484548756632}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05434437602525577, "usage": {"input_tokens": 509, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2dcbafb6fff85371dbe615870f71594cf89d608de4058421fe67cbb39246408f", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-05", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-05", "ts": 1790809217.401152, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7318060856600669, "no": 0.2681939143399331}, "probs_as_returned": {"yes": 0.7318060856600669, "no": 0.2681939143399331}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05498516495572403, "usage": {"input_tokens": 471, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7e976c03966c1a860a1a008a37b17c750fbf1d8358a2859f5ae84b866f096879", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-07", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-07", "ts": 1790809217.4672627, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.2935240287234267, "no": 0.7064759712765734}, "probs_as_returned": {"yes": 0.2935240287234267, "no": 0.7064759712765734}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05443667195504531, "usage": {"input_tokens": 435, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "8fd2009111cdc8dcb0dd58ee42329875c879814cb4e01b92bca4b2985c55e728", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-08", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-08", "ts": 1790809282.2536178, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.45308504361568214, "no": 0.5469149563843179}, "probs_as_returned": {"yes": 0.45308504361568214, "no": 0.5469149563843179}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 64.772698907007, "usage": {"input_tokens": 430, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "1e888421720e1717a354bd055829093d42141019b192c60eefd138bc8c2dc4b4", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-09", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-09", "ts": 1790809282.3274276, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.7557061813028185, "no": 0.24429381869718148}, "probs_as_returned": {"yes": 0.7557061813028185, "no": 0.24429381869718148}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.061641943000722677, "usage": {"input_tokens": 460, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "0b4f67bfea1b71b7f5946de0d972c5a8326907a9ce41106130d4292c465611a3", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-10", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-10", "ts": 1790809282.3979726, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.8527907640733415, "no": 0.14720923592665847}, "probs_as_returned": {"yes": 0.8527907640733415, "no": 0.14720923592665847}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.05935481999767944, "usage": {"input_tokens": 489, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "173157dd6f9297c0221f07c492b47cfa7d4cfa285b6fc644572898b0c2bc4f34", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-11", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-11", "ts": 1790809282.4738345, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.007439008670580321, "no": 0.9925609913294197}, "probs_as_returned": {"yes": 0.007439008670580321, "no": 0.9925609913294197}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06321616697823629, "usage": {"input_tokens": 460, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "0464578f67b5e83f1941683f9801f23075da04ab0bbe6c946d0d00f3c5d1b97a", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-13", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-13", "ts": 1790809282.5479946, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "no", "ordinal_ev": null, "probs": {"yes": 0.10012611628383591, "no": 0.8998738837161641}, "probs_as_returned": {"yes": 0.10012611628383591, "no": 0.8998738837161641}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06192930898396298, "usage": {"input_tokens": 450, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "76ccfdd5d74a5567549b49ad81fdd9c3935e600bebdedf448ba172e76f5cb523", "runtime": null} +{"task_id": "hard-sol-c-judge_hard-15", "family": "judge_hard", "split": "public", "group": "hard-sol-c-judge_hard-15", "ts": 1790809282.6247392, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "yes", "ordinal_ev": null, "probs": {"yes": 0.788764540067536, "no": 0.21123545993246395}, "probs_as_returned": {"yes": 0.788764540067536, "no": 0.21123545993246395}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.06498555600410327, "usage": {"input_tokens": 405, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "78faad73ada646fd4accb4c364b94b0eb8bf24dcc9d38ae20cc32fb5b89cd39a", "runtime": null} +{"task_id": "hard-sol-c-multi_hop-07", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-07", "ts": 1790809282.7743735, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "programme_review", "ordinal_ev": null, "probs": {"partial_release": 0.14708474951632727, "programme_review": 0.5492223588263985, "reject_draw": 0.15660814214094693, "release_draw": 0.14708474951632727}, "probs_as_returned": {"partial_release": 0.14708474951632727, "programme_review": 0.5492223588263985, "reject_draw": 0.15660814214094693, "release_draw": 0.14708474951632727}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.13669061998371035, "usage": {"input_tokens": 2615, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "b009bb49c6045e702541f0c22b640bc79250d2bad25e4945a915ab2c3f02af64", "runtime": null} +{"task_id": "hard-sol-c-multi_hop-09", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-09", "ts": 1790809282.9269676, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "bus_substitution", "ordinal_ev": null, "probs": {"bus_substitution": 0.8215092246509226, "call_fen_cross": 0.02954865144756523, "pass_fen_cross": 0.11748026711353574, "terminate_oakmere": 0.03146185678797637}, "probs_as_returned": {"bus_substitution": 0.8215092246509226, "call_fen_cross": 0.02954865144756523, "pass_fen_cross": 0.11748026711353574, "terminate_oakmere": 0.03146185678797637}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.1379888139781542, "usage": {"input_tokens": 2637, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "2f13b7dd966bcaac598dcdbd77302a4bb01ee88641c4e151509949e6bd12f701", "runtime": null} +{"task_id": "hard-sol-c-multi_hop-11", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-11", "ts": 1790809283.0873387, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "coverage_excluded", "ordinal_ev": null, "probs": {"adjuster_referral": 0.06012453285478484, "coverage_confirmed": 0.04678050972661249, "coverage_excluded": 0.7873470051099877, "documentation_needed": 0.10574795230861497}, "probs_as_returned": {"adjuster_referral": 0.06012453285478484, "coverage_confirmed": 0.04678050972661249, "coverage_excluded": 0.7873470051099877, "documentation_needed": 0.10574795230861497}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.14283783501014113, "usage": {"input_tokens": 2715, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "bf33d93b355243f608347631c97ace0acab103d95c0441ff1f048ae045fb1e6c", "runtime": null} +{"task_id": "hard-sol-c-multi_hop-12", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-12", "ts": 1790809283.162217, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "conservation_hold", "ordinal_ev": null, "probs": {"conservation_hold": 0.9140935257634624, "deny_request": 0.024026050062128126, "digital_copy_only": 0.029001632737729627, "supervised_delivery": 0.03287879143667978}, "probs_as_returned": {"conservation_hold": 0.9140935257634624, "deny_request": 0.024026050062128126, "digital_copy_only": 0.029001632737729627, "supervised_delivery": 0.03287879143667978}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.0635758470161818, "usage": {"input_tokens": 658, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "7cbaa24eb22d1ee944fa05fcecdaf13a697a13a70488533297f978b5a748e0aa", "runtime": null} +{"task_id": "hard-sol-c-multi_hop-13", "family": "multi_hop", "split": "public", "group": "hard-sol-c-multi_hop-13", "ts": 1790809283.3176303, "status": "ok", "ok": true, "valid": true, "correct": true, "predicted": "normal_release", "ordinal_ev": null, "probs": {"engineering_review": 0.022049499850414715, "normal_release": 0.9510435155444504, "rework": 0.011774210282422209, "scrap_lot": 0.015132774322712624}, "probs_as_returned": {"engineering_review": 0.022049499850414715, "normal_release": 0.9510435155444504, "rework": 0.011774210282422209, "scrap_lot": 0.015132774322712624}, "strict_valid": true, "renormalized": false, "probs_source": "native", "model": "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd", "error": null, "schema_error": null, "status_code": 200, "latency_s": 0.14195202803239226, "usage": {"input_tokens": 2683, "output_tokens": 1, "decision_count": 1}, "cost_usd": 0.0, "cost_basis": "zero_route_fee_compute_excluded", "reserved_usd": 0.02, "charged_usd": 0.0, "raw_sha256": "314f3659b59641054ade97345c201ca9010b31f2329b958a978047033c684002", "runtime": null} diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/jb-summary-0.1.3.json b/services/intern-decision-serve/acceptance/systemone-2026-09-30/jb-summary-0.1.3.json new file mode 100644 index 0000000..0292857 --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/jb-summary-0.1.3.json @@ -0,0 +1,2141 @@ +{ + "accuracy": 0.8744588744588745, + "brier_mean": 0.17806609620099598, + "calibration_n": 231, + "complete": true, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.4, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": 0.3623595989308779, + "n": 10 + }, + { + "accuracy": 0.0, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": 0.46124870941679735, + "n": 5 + }, + { + "accuracy": 0.8, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5637253221582417, + "n": 10 + }, + { + "accuracy": 0.65, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.648800445448587, + "n": 20 + }, + { + "accuracy": 0.7391304347826086, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7523100109863671, + "n": 23 + }, + { + "accuracy": 0.9117647058823529, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8641129171311692, + "n": 34 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9627515382033693, + "n": 129 + } + ], + "ece": 0.051072415884039386, + "n": 231 + }, + "latency": { + "n": 231, + "p50_s": 0.05428842600667849, + "p95_s": 0.14888899750076234 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "ledger_charged_usd": 0.0, + "macro_accuracy": 0.8725744098809114, + "model_identities": [ + "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd" + ], + "n_attempted": 231, + "n_correct": 202, + "n_planned": 231, + "n_renormalized": 0, + "n_scorable": 231, + "n_valid": 231, + "operational_success": 1.0, + "ordinal_mae": 0.26234367203481096, + "paraphrase_consistency": { + "agree": 35, + "agreement": 0.9722222222222222, + "both_correct_rate_all_pairs": 0.9722222222222222, + "both_valid": 36, + "pairs": 36 + }, + "per_family": { + "adequacy": { + "accuracy": 1.0, + "brier_mean": 0.04603689738961828, + "calibration_n": 12, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5930110454411601, + "n": 1 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7557061813028185, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8878357373215198, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9520119763886462, + "n": 8 + } + ], + "ece": 0.104959623958651, + "n": 12 + }, + "latency": { + "n": 12, + "p50_s": 0.052879591967212036, + "p95_s": 0.054876283335033804 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 12, + "n_correct": 12, + "n_planned": 12, + "n_renormalized": 0, + "n_scorable": 12, + "n_valid": 12, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "adversarial": { + "accuracy": 1.0, + "brier_mean": 0.011617825043304442, + "calibration_n": 6, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8627679157764538, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9629914990167525, + "n": 4 + } + ], + "ece": 0.07041636206334703, + "n": 6 + }, + "latency": { + "n": 6, + "p50_s": 0.05504016700433567, + "p95_s": 0.05621324274397921 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 6, + "n_correct": 6, + "n_planned": 6, + "n_renormalized": 0, + "n_scorable": 6, + "n_valid": 6, + "operational_success": 1.0, + "ordinal_mae": 0.10376329334202228, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "ambiguous": { + "accuracy": 0.7142857142857143, + "brier_mean": 0.3223035707740803, + "calibration_n": 7, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.3333333333333333, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6479833643674017, + "n": 3 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9463672181616865, + "n": 4 + } + ], + "ece": 0.16549731720792277, + "n": 7 + }, + "latency": { + "n": 7, + "p50_s": 0.06194691703421995, + "p95_s": 0.0629515228967648 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 7, + "n_correct": 5, + "n_planned": 7, + "n_renormalized": 0, + "n_scorable": 7, + "n_valid": 7, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "extraction": { + "accuracy": 1.0, + "brier_mean": 0.0019479746093977082, + "calibration_n": 24, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9683153887441366, + "n": 24 + } + ], + "ece": 0.031684611255863415, + "n": 24 + }, + "latency": { + "n": 24, + "p50_s": 0.053058729477925226, + "p95_s": 0.0536909365124302 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 24, + "n_correct": 24, + "n_planned": 24, + "n_renormalized": 0, + "n_scorable": 24, + "n_valid": 24, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "fact": { + "accuracy": 1.0, + "brier_mean": 0.00253357674402684, + "calibration_n": 12, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9707540110368739, + "n": 12 + } + ], + "ece": 0.029245988963126135, + "n": 12 + }, + "latency": { + "n": 12, + "p50_s": 0.05440999200800434, + "p95_s": 0.0573352141684154 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 12, + "n_correct": 12, + "n_planned": 12, + "n_renormalized": 0, + "n_scorable": 12, + "n_valid": 12, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "intent": { + "accuracy": 0.9583333333333334, + "brier_mean": 0.04236379090188356, + "calibration_n": 24, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.0, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": 0.4791247461286767, + "n": 1 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.697318944283325, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7504260155024499, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8623501745620707, + "n": 3 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.97124439087109, + "n": 17 + } + ], + "ece": 0.0909478584310712, + "n": 24 + }, + "latency": { + "n": 24, + "p50_s": 0.053307871479773894, + "p95_s": 0.060923036502208545 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 24, + "n_correct": 23, + "n_planned": 24, + "n_renormalized": 0, + "n_scorable": 24, + "n_valid": 24, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "judge_hard": { + "accuracy": 0.9411764705882353, + "brier_mean": 0.1769820408934362, + "calibration_n": 17, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5469149563843179, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6659927647326704, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7429117727934124, + "n": 5 + }, + { + "accuracy": 0.8, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8717155645981629, + "n": 5 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9748315019610428, + "n": 5 + } + ], + "ece": 0.15040904282950293, + "n": 17 + }, + "latency": { + "n": 17, + "p50_s": 0.054366590979043394, + "p95_s": 13.006528226204635 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 17, + "n_correct": 16, + "n_planned": 17, + "n_renormalized": 0, + "n_scorable": 17, + "n_valid": 17, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "long_policy": { + "accuracy": 0.6842105263157895, + "brier_mean": 0.488183253400819, + "calibration_n": 19, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.5, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": 0.32987837012316157, + "n": 2 + }, + { + "accuracy": 0.0, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": 0.4247953847281636, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5930110454411601, + "n": 1 + }, + { + "accuracy": 0.6666666666666666, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6374422769566587, + "n": 3 + }, + { + "accuracy": 0.5, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7434209921956753, + "n": 4 + }, + { + "accuracy": 0.8333333333333334, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8591073374831182, + "n": 6 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9264976568409564, + "n": 2 + } + ], + "ece": 0.13342281306158954, + "n": 19 + }, + "latency": { + "n": 19, + "p50_s": 0.14695013902382925, + "p95_s": 0.18794695567921732 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 19, + "n_correct": 13, + "n_planned": 19, + "n_renormalized": 0, + "n_scorable": 19, + "n_valid": 19, + "operational_success": 1.0, + "ordinal_mae": 0.8438052410179581, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "multi_hop": { + "accuracy": 0.8333333333333334, + "brier_mean": 0.2821094188229273, + "calibration_n": 18, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": 0.3946907024781775, + "n": 1 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.75, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5453134547809111, + "n": 4 + }, + { + "accuracy": 1.0, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6537803635138819, + "n": 2 + }, + { + "accuracy": 0.5, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7674221407596936, + "n": 4 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8512239723447685, + "n": 5 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9325685206539565, + "n": 2 + } + ], + "ece": 0.22582924507652405, + "n": 18 + }, + "latency": { + "n": 18, + "p50_s": 0.14008955997996964, + "p95_s": 0.15341312710661442 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 18, + "n_correct": 15, + "n_planned": 18, + "n_renormalized": 0, + "n_scorable": 18, + "n_valid": 18, + "operational_success": 1.0, + "ordinal_mae": 0.3268150682115487, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "ordinal": { + "accuracy": 1.0, + "brier_mean": 0.006310346320679678, + "calibration_n": 12, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8500853530913438, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.956899632823367, + "n": 11 + } + ], + "ece": 0.05200155715430161, + "n": 12 + }, + "latency": { + "n": 12, + "p50_s": 0.053522854985203594, + "p95_s": 0.05490196594328153 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 12, + "n_correct": 12, + "n_planned": 12, + "n_renormalized": 0, + "n_scorable": 12, + "n_valid": 12, + "operational_success": 1.0, + "ordinal_mae": 0.0531333548567569, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "policy": { + "accuracy": 1.0, + "brier_mean": 0.020422084182937634, + "calibration_n": 12, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7557061813028185, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8742406957013814, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9513615324190499, + "n": 9 + } + ], + "ece": 0.07779655296024748, + "n": 12 + }, + "latency": { + "n": 12, + "p50_s": 0.053544344496913254, + "p95_s": 0.05435312601330225 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 12, + "n_correct": 12, + "n_planned": 12, + "n_renormalized": 0, + "n_scorable": 12, + "n_valid": 12, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "probability": { + "accuracy": 0.7, + "brier_mean": 0.46568152828081544, + "calibration_n": 10, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5777854893190272, + "n": 1 + }, + { + "accuracy": 0.8, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6325445353204342, + "n": 5 + }, + { + "accuracy": 0.3333333333333333, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7561351202525173, + "n": 3 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8908360355378411, + "n": 1 + }, + { + "accuracy": null, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": null, + "n": 0 + } + ], + "ece": 0.26370611592985127, + "n": 10 + }, + "latency": { + "n": 10, + "p50_s": 0.06932540450361557, + "p95_s": 0.0942387860675808 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 10, + "n_correct": 7, + "n_planned": 10, + "n_renormalized": 0, + "n_scorable": 10, + "n_valid": 10, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "routing": { + "accuracy": 1.0, + "brier_mean": 0.041188901489954015, + "calibration_n": 12, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6801954980304856, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7877520218832728, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8825011701644436, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9364805526689252, + "n": 7 + } + ], + "ece": 0.12762423108703272, + "n": 12 + }, + "latency": { + "n": 12, + "p50_s": 0.052542628545779735, + "p95_s": 0.05301162832183763 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 12, + "n_correct": 12, + "n_planned": 12, + "n_renormalized": 0, + "n_scorable": 12, + "n_valid": 12, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "routing_hard": { + "accuracy": 1.0, + "brier_mean": 0.0051631143261297256, + "calibration_n": 5, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9423656177532307, + "n": 5 + } + ], + "ece": 0.05763438224676931, + "n": 5 + }, + "latency": { + "n": 5, + "p50_s": 0.054628726036753505, + "p95_s": 0.05692764220293611 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 5, + "n_correct": 5, + "n_planned": 5, + "n_renormalized": 0, + "n_scorable": 5, + "n_valid": 5, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "temporal_numeric": { + "accuracy": 0.3333333333333333, + "brier_mean": 0.7767759878778288, + "calibration_n": 15, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.2857142857142857, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": 0.36702122094061124, + "n": 7 + }, + { + "accuracy": 0.0, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": 0.4681313198099223, + "n": 2 + }, + { + "accuracy": 0.5, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5726384329365539, + "n": 2 + }, + { + "accuracy": 0.0, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6472762711048763, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7318060856600669, + "n": 1 + }, + { + "accuracy": 0.5, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8512242138235586, + "n": 2 + }, + { + "accuracy": null, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": null, + "n": 0 + } + ], + "ece": 0.2179071110112772, + "n": 15 + }, + "latency": { + "n": 15, + "p50_s": 0.06334954197518528, + "p95_s": 0.08096886847633866 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 15, + "n_correct": 5, + "n_planned": 15, + "n_renormalized": 0, + "n_scorable": 15, + "n_valid": 15, + "operational_success": 1.0, + "ordinal_mae": 1.102096986624212, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "tool_selection": { + "accuracy": 1.0, + "brier_mean": 0.00027882058159227585, + "calibration_n": 12, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9859023175222424, + "n": 12 + } + ], + "ece": 0.014097682477757556, + "n": 12 + }, + "latency": { + "n": 12, + "p50_s": 0.05312584349303506, + "p95_s": 0.05412412802688778 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 12, + "n_correct": 12, + "n_planned": 12, + "n_renormalized": 0, + "n_scorable": 12, + "n_valid": 12, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "tradeoff": { + "accuracy": 0.6666666666666666, + "brier_mean": 0.3172021465853196, + "calibration_n": 6, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.0, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.6392348025938888, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7449709949864798, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8562976654414707, + "n": 2 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9682766780586276, + "n": 1 + } + ], + "ece": 0.30877110020995474, + "n": 6 + }, + "latency": { + "n": 6, + "p50_s": 0.06751805800013244, + "p95_s": 0.06984318800095934 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 6, + "n_correct": 4, + "n_planned": 6, + "n_renormalized": 0, + "n_scorable": 6, + "n_valid": 6, + "operational_success": 1.0, + "ordinal_mae": null, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + }, + "trap": { + "accuracy": 0.875, + "brier_mean": 0.0928845979050508, + "calibration_n": 8, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.0, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": 0.4660607766073019, + "n": 1 + }, + { + "accuracy": null, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 1.0, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8527907640733415, + "n": 1 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9658182629839495, + "n": 6 + } + ], + "ece": 0.10229505432878294, + "n": 8 + }, + "latency": { + "n": 8, + "p50_s": 0.055600558494916186, + "p95_s": 0.05691805843380279 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 8, + "n_correct": 7, + "n_planned": 8, + "n_renormalized": 0, + "n_scorable": 8, + "n_valid": 8, + "operational_success": 1.0, + "ordinal_mae": 0.8643000081318148, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + } + }, + "price_per_1000_decisions_usd": 0.0, + "probability_sources": [ + "native" + ], + "schema_validity": 1.0, + "schema_validity_strict": 1.0, + "splits": { + "public": { + "accuracy": 0.8744588744588745, + "brier_mean": 0.17806609620099598, + "calibration_n": 231, + "cost_basis": [ + "zero_route_fee_compute_excluded" + ], + "coverage": 1.0, + "ece": { + "bins": [ + { + "accuracy": null, + "hi": 0.1, + "lo": 0.0, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.2, + "lo": 0.1, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": null, + "hi": 0.3, + "lo": 0.2, + "mean_confidence": null, + "n": 0 + }, + { + "accuracy": 0.4, + "hi": 0.4, + "lo": 0.3, + "mean_confidence": 0.3623595989308779, + "n": 10 + }, + { + "accuracy": 0.0, + "hi": 0.5, + "lo": 0.4, + "mean_confidence": 0.46124870941679735, + "n": 5 + }, + { + "accuracy": 0.8, + "hi": 0.6, + "lo": 0.5, + "mean_confidence": 0.5637253221582417, + "n": 10 + }, + { + "accuracy": 0.65, + "hi": 0.7, + "lo": 0.6, + "mean_confidence": 0.648800445448587, + "n": 20 + }, + { + "accuracy": 0.7391304347826086, + "hi": 0.8, + "lo": 0.7, + "mean_confidence": 0.7523100109863671, + "n": 23 + }, + { + "accuracy": 0.9117647058823529, + "hi": 0.9, + "lo": 0.8, + "mean_confidence": 0.8641129171311692, + "n": 34 + }, + { + "accuracy": 1.0, + "hi": 1.0, + "lo": 0.9, + "mean_confidence": 0.9627515382033693, + "n": 129 + } + ], + "ece": 0.051072415884039386, + "n": 231 + }, + "latency": { + "n": 231, + "p50_s": 0.05428842600667849, + "p95_s": 0.14888899750076234 + }, + "latency_failures": { + "n": 0, + "p50_s": null, + "p95_s": null + }, + "n_attempted": 231, + "n_correct": 202, + "n_planned": 231, + "n_renormalized": 0, + "n_scorable": 231, + "n_valid": 231, + "operational_success": 1.0, + "ordinal_mae": 0.26234367203481096, + "price_per_1000_decisions_usd": 0.0, + "schema_validity": 1.0, + "schema_validity_strict": 1.0 + } + }, + "version": "jevbench-v1" +} diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/random-validation.txt b/services/intern-decision-serve/acceptance/systemone-2026-09-30/random-validation.txt new file mode 100644 index 0000000..44154ec --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/random-validation.txt @@ -0,0 +1,14 @@ + tokens~ tokens wall_s + 8285 8280 0.45 + 11080 11075 0.62 + 14286 14285 0.84 + 16164 16160 0.97 + 17447 17445 1.05 + 21661 21660 1.35 + 21966 21965 1.37 + 23112 23110 1.45 + 24358 24355 1.54 + 24567 24565 1.56 + 24911 24910 1.58 + 31822 31820 2.09 +worst wall 2.09 s — every call warm diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/warmup-cold.txt b/services/intern-decision-serve/acceptance/systemone-2026-09-30/warmup-cold.txt new file mode 100644 index 0000000..7322571 --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/warmup-cold.txt @@ -0,0 +1,20 @@ +endpoint=http://intern-decision.fv.internal:8033 max_tokens=32768 bucket_width=2048 buckets=16 +calibrated: input_tokens = 5.0000*reps + 183.0 + bucket<= tokens wall_s note + 2048 1988 0.11 + 4096 4038 6.53 + 6144 6083 0.31 + 8192 8133 6.75 + 10240 10178 7.00 + 12288 12228 7.16 + 14336 14278 7.28 + 16384 16323 7.52 + 18432 18373 7.72 + 20480 20418 7.92 + 22528 22468 8.08 + 24576 24518 8.24 + 26624 26563 8.40 + 28672 28613 8.60 + 30720 30658 8.77 + 32768 32708 8.90 +total 109.3 s over 16 buckets diff --git a/services/intern-decision-serve/acceptance/systemone-2026-09-30/warmup-warm.txt b/services/intern-decision-serve/acceptance/systemone-2026-09-30/warmup-warm.txt new file mode 100644 index 0000000..4c42701 --- /dev/null +++ b/services/intern-decision-serve/acceptance/systemone-2026-09-30/warmup-warm.txt @@ -0,0 +1,20 @@ +endpoint=http://intern-decision.fv.internal:8033 max_tokens=32768 bucket_width=2048 buckets=16 +calibrated: input_tokens = 5.0000*reps + 183.0 + bucket<= tokens wall_s note + 2048 1988 0.11 + 4096 4038 0.20 + 6144 6083 0.31 + 8192 8133 0.43 + 10240 10178 0.56 + 12288 12228 0.70 + 14336 14278 0.84 + 16384 16323 0.97 + 18432 18373 1.12 + 20480 20418 1.26 + 22528 22468 1.40 + 24576 24518 1.55 + 26624 26563 1.71 + 28672 28613 1.85 + 30720 30658 2.01 + 32768 32708 2.17 +total 17.2 s over 16 buckets diff --git a/services/intern-decision-serve/pyproject.toml b/services/intern-decision-serve/pyproject.toml index ffb00ab..8081980 100644 --- a/services/intern-decision-serve/pyproject.toml +++ b/services/intern-decision-serve/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "intern-decision-serve" -version = "0.1.2" +version = "0.1.3" description = "Intern-Decision-4B (its own inference.py) behind semif-serve's HTTP surface" requires-python = ">=3.12" dependencies = [ diff --git a/services/intern-decision-serve/uv.lock b/services/intern-decision-serve/uv.lock index fb89c74..1b04bf6 100644 --- a/services/intern-decision-serve/uv.lock +++ b/services/intern-decision-serve/uv.lock @@ -284,7 +284,7 @@ wheels = [ [[package]] name = "intern-decision-serve" -version = "0.1.2" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "fastapi" }, diff --git a/stacks/intern-decision/README.md b/stacks/intern-decision/README.md index cbf6ac3..e6ef451 100644 --- a/stacks/intern-decision/README.md +++ b/stacks/intern-decision/README.md @@ -102,18 +102,27 @@ Measured on the live service (per-process nvidia-smi every 0.1 s; single `noul` Spare at the limit: 217 MiB. JevBench v1.2.16 through `/v1/systemone` after the change: 202/231, with 0 answer and 0 probability diffs against the bench's r1..r4. `/decide` is unchanged. -⚠ **Cold-shape latency:** the first call in a new length bucket after a (re)start costs ~6.5 s extra -(3,001 and 4,000 words were slow; 3,002–3,500 and 4,097–5,000 were not). The fast kernels autotune per shape -bucket. Warm calls are as tabled. - - **How long the warm state lasts (measured 1541):** it lives in the Triton cache at `/tmp/triton-cache`, including - fla's `*.autotune.json`, in the container's writable layer. It SURVIVES `docker restart` (and so host reboots): - after a restart, a seen 32k bucket answered in 2.0 s. It is LOST on recreate (`compose up -d` after a change, a - deploy, an image upgrade). An unseen bucket after the restart still cost +6.4 s (12,187 tokens), which is the - positive control. - - **Buckets:** every observation so far fits buckets of 2,048 tokens (inferred from ~20 calls, not proven), so ~16 - buckets up to 32,768. A full warm-up is estimated at ~2 min, once per container. - - **Durable fix (not done):** put `/tmp/triton-cache` on a named volume so that recreates keep it, and run one - ~2 min warm-up across the buckets after an image change (a new triton/fla version keys new entries). +⚠ **Cold-shape latency (MEASURED 2026-09-30, was inferred):** the fast kernels (triton + fla) autotune +per shape bucket. **Buckets are 2,048 tokens wide** — proven: after warming one call per bucket, 12 +calls at random sizes spread 8k–32k were all warm (worst 2.09 s, the same curve as the warmed sizes; +no +6 s), and every cold call that crossed into its own bucket cost 6.5–9 s while same-bucket repeats +cost <0.3 s. **16 buckets** to MAX_TOKENS=32,768. A full cold warm-up took **109 s** (16 buckets; the +service's own startup warm-up and the script's two calibration probes warm 3 of them along the way; +a second run totals 17 s). Per-bucket cold cost grows with size: 6.5 s at 4k → 8.9 s at 32k. + - **Durable now:** the autotune cache (TRITON_CACHE_DIR=/tmp/triton-cache, incl. fla's + *.autotune.json) lives on the named volume `intern-decision_triton-cache`, mounted at a mount + point baked into the image owned by 10001 (a fresh named volume inherits it; without that, the + volume is root-owned and Triton cannot write). Proven: 3,200+ files owned by uid 10001 in the + volume; after `up -d --force-recreate` a previously warmed 32k call answered in 2.11 s. + - **When to warm:** run `scripts/intern-decision-warmup` (on nh3-dev; token from the vault) after + an IMAGE CHANGE — a new triton/fla version keys new cache entries. Not needed after ordinary + recreates. The script reads MAX_TOKENS from /health, aims each bucket just under its edge with a + two-point live calibration (single-probe "calibration" oscillates ±500 tokens around the edge), + prints per-bucket wall times, and exits non-zero if any bucket was missed. + - Cold autotune also costs a few MiB of transient scratch: GPU 1 per-process peak measured + **15,218 MiB warm** at the 32k limit (budget 15,220), but **15,224** during a COLD 32k autotune. + The volume means the cold path now happens only after an image change — worth knowing when + GPU 1 headroom is being read during a deploy. The table below is the ORIGINAL budget (9.0 GiB / 7,168 tokens, beside scriberr) and is kept for history. @@ -253,6 +262,9 @@ scripts/deploy-stack.sh fv-ml1 intern-decision # on fv-ml1: .env is infra-ops's, mode 0600. Rewrite it IN PLACE to keep the owner and mode. cd /opt/docker/compose/intern-decision && new=$(sed 's/^IMAGE=.*/IMAGE=intern-decision-serve:X.Y.Z/' .env) \ && printf '%s\n' "$new" > .env && docker compose config -q && docker compose up -d +# IMAGE CHANGE ONLY (not ordinary recreates — the triton-cache volume carries those): +# warm every length bucket once the container is healthy (~2 min, prints per-bucket times): +scripts/intern-decision-warmup # token from vault; MAX_TOKENS read from /health ``` **Before any deploy onto GPU 1**, check two things. If either fails, stop; do not squeeze diff --git a/stacks/intern-decision/compose.yaml b/stacks/intern-decision/compose.yaml index 99fd6fb..f4c9fb9 100644 --- a/stacks/intern-decision/compose.yaml +++ b/stacks/intern-decision/compose.yaml @@ -37,6 +37,10 @@ services: volumes: # Pinned weights AND the checkpoint's inference.py, read offline. Never downloads. - /tank/aimodels/huggingface:/hf:ro + # Triton/fla autotune cache (README "Cold-shape latency"): the ~6.5 s first-call-per-bucket + # autotune results survive RECREATES (deploys, image upgrades, .env edits), not just restarts. + # The mount point exists in the image owned by 10001, so the fresh volume is intern-writable. + - triton-cache:/tmp/triton-cache deploy: resources: reservations: @@ -64,3 +68,7 @@ networks: tnet: name: traefik-net external: true + +volumes: + # -> intern-decision_triton-cache on the host + triton-cache: diff --git a/stacks/scriberr/patches/proposed/0002-parakeet-model-path-and-gap-retry.patch b/stacks/scriberr/patches/0002-parakeet-model-path-and-gap-retry.patch similarity index 100% rename from stacks/scriberr/patches/proposed/0002-parakeet-model-path-and-gap-retry.patch rename to stacks/scriberr/patches/0002-parakeet-model-path-and-gap-retry.patch