vh
a262477a61
stacks/intern-decision: compose (GPU 1, :8033, hard VRAM cap as the single .env knob, healthcheck, Homepage group 'AI - Eval & Retrieval'), .env.example and README. dns: intern-decision.fv.internal -> fv-ml1 (synced to ana/esh/nh3). acceptance on fv-ml1 GPU 3, 3 fresh processes: bit-identical to the Jev bench's native rows (pooled 240/259, Wyrd 79/84, 0/560 flips, Δp 0), negative control 10/122/14, 0 flips across restarts; largest accepted request 200 at a 10,134 MiB card peak under a 9.25 GiB cap; 503 and recovery proven at a tight cap. GPU 1 deploy held: nvidia-smi Free on GPU 1 is 15,442 MiB.
1 line
290 KiB
JSON
1 line
290 KiB
JSON
{"label": "oomcap", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:34:14Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.23722597211599, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 37.54}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41475101560354, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.4}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60182298719883, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.98}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.148128962144256, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.33}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.222439888864756, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.57}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39509196020663, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.7}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52464312128723, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.76}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09907806478441, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.34}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.381431087851524, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.51}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8156349286437, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.09}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.320130970329046, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.58}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.948106018826365, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 34.27}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38309186883271, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.64}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81180509738624, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.12}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30958001688123, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 34.6}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9753569830209, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.26}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30314993299544, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.56}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03837702050805, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.24}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42932202294469, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.62}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09882707335055, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.3}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38433099351823, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 34.71}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02024603262544, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.16}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56383387371898, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.64}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219228921458125, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.51}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47240197844803, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.75}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.92493499815464, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 39.96}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.093718932941556, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 35.28}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98281797207892, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 35.3}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.509343983605504, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 35.84}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.301220862194896, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.58}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28869201242924, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 35.49}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39951287955046, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 35.58}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6781960260123, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 35.93}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.41461311094463, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 35.64}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32499100826681, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 35.62}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.973105976358056, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.27}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62668394483626, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.75}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89747606404126, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.11}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41311212070286, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.62}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40755212493241, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.46}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.56027405709028, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 35.78}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.374661071226, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.48}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13059797883034, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.49}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.281839875504375, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.45}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92189690284431, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 35.03}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85469483025372, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.08}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.292209919542074, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.38}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13972999341786, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 35.26}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.809207966551185, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 35.95}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.16693997941911, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 35.48}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.27066090889275, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 35.61}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.10907993465662, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 35.36}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.68022607639432, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 35.79}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.39140199124813, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 35.64}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.41776307672262, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 35.71}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.45210303552449, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 35.74}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.75626716762781, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 35.93}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.37587311863899, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 35.62}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.55350497178733, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 35.88}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.278331980109215, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 35.56}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.541654823347926, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 35.77}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28664100728929, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 35.49}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.38413301855326, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 35.62}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.17991097457707, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 35.47}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51268393918872, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 35.63}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.36064296774566, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 35.67}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.62329595722258, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 35.93}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.19341096095741, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 35.5}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59798493795097, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 35.92}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.12647983431816, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 35.38}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.12159102037549, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 36.13}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.213969906792045, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 35.37}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.64391497336328, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 35.8}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15719997510314, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 35.35}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.37492293678224, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 35.47}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5087039321661, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 35.49}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.87483913078904, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 36.88}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.701486075297, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 35.7}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.76746702007949, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 35.9}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.46478299237788, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 35.48}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.500963943079114, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 35.58}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.355602882802486, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 35.46}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.43852295540273, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 35.48}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.54936407878995, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 35.58}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.746176986023784, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 35.88}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.609656039625406, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 35.76}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7645079754293, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 35.9}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.484642980620265, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 35.52}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7450170237571, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 36.01}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.58252498321235, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 35.7}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.53966388851404, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 35.65}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.532903883606195, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 35.72}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.45397406257689, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 35.55}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.18007405102253, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 37.14}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.83250686340034, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 35.77}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.141730008646846, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 35.31}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.711885921657085, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 35.86}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.2578510325402, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 35.39}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.44646317884326, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 35.6}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.643134128302336, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.72}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.369539961218834, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.54}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08837786689401, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.19}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33514994382858, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.49}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.929765017703176, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.11}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53050300292671, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.63}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34834096394479, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.53}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19118796288967, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.32}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.505812011659145, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.59}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1606289409101, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.36}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37414092756808, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.43}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.387041142210364, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.52}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.097027990967035, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.22}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34170086681843, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.46}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01535705476999, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.22}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63216291554272, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.81}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2465499676764, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.33}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30636090040207, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.45}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14422881603241, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.19}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42505190894008, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.47}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90437483601272, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.11}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.274840055033565, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.5}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0169070083648, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.22}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24615997634828, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.42}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31174999848008, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.47}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27444005198777, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.37}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96497599594295, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.15}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3413001652807, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.43}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17161901295185, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.2}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.056667963042855, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.25}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00762614607811, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.16}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.289900006726384, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.41}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48924215324223, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.78}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.277380004525185, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.33}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32143994420767, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.45}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3251608107239, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.5}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.320510018616915, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.53}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.399220978841186, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.77}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18486800789833, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.44}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66330401413143, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.84}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19314909540117, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.5}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24647895433009, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.5}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94557584822178, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.13}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.162937922403216, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.51}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01731702312827, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.33}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.623084150254726, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.96}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96165606379509, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 34.29}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29939996264875, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.52}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.271609995514154, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.57}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9182659778744, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.24}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.558943033218384, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 34.77}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53821295686066, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 34.7}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12768782861531, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.26}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52825199626386, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 34.72}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.092468002811074, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.41}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.181489169597626, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.55}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44786186143756, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 34.73}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35298989340663, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.69}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06193692050874, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.38}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.01111094839871, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 35.85}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.68903615511954, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.84}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.642693845555186, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 34.81}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49979310669005, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 34.72}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31518005952239, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 34.44}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.553382804617286, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 34.71}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35398990102112, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 34.61}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.049086997285485, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.32}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39555086940527, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 34.61}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.181448889896274, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.48}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27221884764731, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.47}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366241216659546, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 34.56}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.330060033127666, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 34.56}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.488111993297935, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 34.56}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.277020048350096, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 34.54}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45296294800937, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 34.6}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.453820928931236, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 34.63}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37088106572628, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.53}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34380006790161, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 34.57}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54039295017719, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 34.8}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20557899214327, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 34.46}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5595330260694, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 34.72}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.394262151792645, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 34.69}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18284796923399, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 34.48}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.785826018080115, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 34.98}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15310897678137, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.43}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28171996772289, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 34.48}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12688805535436, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 34.38}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66276501119137, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.82}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53853286616504, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 34.71}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36784099601209, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 34.54}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.466422887519, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.59}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27469011209905, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 34.61}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21635905094445, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.48}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44870191439986, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 34.67}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29078103788197, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 34.63}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65176306478679, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 34.66}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.295239977538586, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.53}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76869596913457, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 34.87}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63848403468728, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 34.69}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67106402665377, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 34.82}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.814775947481394, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 34.78}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76543587446213, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.89}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58483310230076, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.61}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71311487071216, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 34.86}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.40134691260755, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 38.53}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2775299474597, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 34.37}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32293006032705, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.45}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.87099686451256, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 34.88}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.351120030507445, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 34.64}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62671397998929, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 34.87}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.993148900568485, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 35.17}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.722705164924264, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.76}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.437052000314, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 34.56}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71920409053564, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 34.89}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22933982498944, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 34.36}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.332889856770635, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 34.53}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.308149971067905, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 34.47}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.554593058303, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.89}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38051187247038, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 34.69}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4844828620553, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 34.78}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.238848861306906, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.56}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.328319856897, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 34.6}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32498991303146, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 34.66}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.504232954233885, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.8}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30669105425477, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.5}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.938406923785806, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 35.25}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63057407923043, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.74}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.274949019774795, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 34.63}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65260404907167, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 34.94}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17986890114844, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 34.49}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01054700650275, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.34}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.894396809861064, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 35.11}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20629890449345, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.54}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25675889290869, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 34.56}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48929291032255, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 34.69}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39581210538745, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 34.75}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.196108140051365, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 34.36}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64725383371115, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 34.86}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.210228852927685, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 34.51}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68860409036279, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 34.87}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47408208437264, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 34.75}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56812307983637, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 34.78}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.394850980490446, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 34.52}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.202089093625546, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 34.45}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38056984730065, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.6}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47487301006913, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 34.63}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.200018065050244, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.34}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63307398185134, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.7}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.132757948711514, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 34.31}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.868306040763855, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 34.87}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06219792366028, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.31}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19212906435132, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 34.31}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53431304357946, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.55}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.19284099154174, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 35.34}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07225690595806, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.16}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.920276867225766, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 35.09}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26090909913182, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.25872088782489, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 35.28}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17040806077421, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.721164990216494, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 34.78}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.370400968939066, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.5}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.944956915453076, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 35.0}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28626994416118, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.45}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.78369491919875, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98966698721051, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.22}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59627300314605, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.77}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23316895775497, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93475707061589, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 35.09}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3845009598881, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.913707084953785, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 35.11}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29499108158052, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.29}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84358687698841, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 35.05}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1690490283072, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.25}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.9963091108948, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 35.03}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3159209266305, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.44}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46665199287236, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 34.52}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00981708243489, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.17}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85625612363219, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 34.98}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29233983904123, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71119401790202, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 34.84}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02673711813986, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.781886057928205, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 34.95}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18265891075134, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.37}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23824995197356, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 35.24}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.416460923850536, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.56}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.9880191758275, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 35.19}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.295280024409294, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5001040995121, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 35.54}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47580200433731, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.57}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.17253094166517, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 35.16}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50711283646524, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.66}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.05902902968228, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 35.2}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39314083941281, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.52}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.18237013183534, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 35.21}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3239500913769, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.967418087646365, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 35.19}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.393131060525775, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.879507057368755, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 35.02}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69618389196694, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.9}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.21244090795517, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 35.41}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19574888236821, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.56}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90636686608195, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 35.22}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37120097503066, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.96049696393311, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 35.26}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.363520823419094, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6785141415894, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 35.01}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54643304273486, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.69}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.20465109311044, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 35.42}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.157318787649274, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93897689320147, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 35.29}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28582989424467, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.51}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.044807966798544, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 35.24}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14055777527392, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.32}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.012698059901595, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 35.04}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.578893126919866, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.76}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98895818181336, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 35.2}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34340099617839, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.54}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.247771095484495, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 35.34}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.392440950497985, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.22628105990589, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 35.32}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27182000875473, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.5}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76667593047023, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 34.91}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.241159938275814, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.59}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66026510857046, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.632894003763795, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76150499470532, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.167889066040516, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.5}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.507982924580574, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 34.85}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43492090143263, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.65}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.882277043536305, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41633193567395, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.75}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77905507013202, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 34.98}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31067990884185, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.5}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.70685498602688, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 34.84}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.138188956305385, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.49}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68459497950971, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.197329103946686, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51575294882059, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28022985160351, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.61}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41095098666847, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 34.73}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06585709005594, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.39}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.85259695723653, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35737991519272, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81310492195189, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 34.86}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3339899815619, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.62}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62741503305733, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34614094719291, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.48}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39145118743181, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 34.63}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.293750094249845, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.36}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58715302683413, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 34.83}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22489904612303, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.5}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56980295293033, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03207685612142, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.22}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54350310191512, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 34.74}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.324680015444756, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.52}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.812156137079, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 34.89}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.6053320299834, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 40.23}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 41.28148593008518, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 37.9}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.29975279606879, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 36.6}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.51967600733042, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 36.69}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.720316952094436, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.94}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69688604027033, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 35.98}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59358311071992, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.814185954630375, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4731719493866, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63327398337424, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.400661036372185, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.72}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56035305559635, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 34.85}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45980188250542, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.65}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63622301630676, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59067296050489, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.73}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66763396561146, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 34.8}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54875296726823, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.78}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60449308902025, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184438202530146, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.763225845992565, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26228000596166, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.46}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.750134943053126, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 34.94}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28356002829969, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.64}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79680607840419, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38389117643237, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.6}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.631102837622166, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 34.81}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.536223186179996, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.62}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62908396311104, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 34.9}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.275920156389475, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.52}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58123307488859, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27397997863591, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.59}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69487398676574, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 34.87}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.185578839853406, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.51}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45308192446828, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 34.66}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00314718298614, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.35}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28927904739976, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 34.6}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96490591391921, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.3}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 42.2695679590106, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 39.43}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.326201098039746, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.49}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.97449001669884, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 36.15}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.183409091085196, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.38}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.18282997235656, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 35.37}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97930695489049, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.17}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6505660135299, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 35.81}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62237308546901, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.82}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.48197404295206, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 35.69}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.457781843841076, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.79}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.84954797104001, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 35.98}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.442742148414254, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.59}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69267599657178, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 35.78}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.218199111521244, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.46}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.70195709168911, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 35.84}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60918392799795, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.78942714072764, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.648963974788785, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.86}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.652455899864435, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63548377901316, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.84}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.62937516532838, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 35.7}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19958895817399, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.48}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.90152811072767, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 36.07}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65335492789745, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.704317063093185, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 35.97}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.5112930778414, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.78}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.81019679829478, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 36.1}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.8879071213305, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.07}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.77075691707432, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 35.99}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34439006075263, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.69}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.20458210632205, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 36.32}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90418687276542, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.94073888659477, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 36.15}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47088206000626, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.65}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.017789997160435, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 36.2}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84730611369014, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.03}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65421493537724, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 35.82}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.079717963933945, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.36}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.807767909020185, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 36.08}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.630253937095404, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.69877592660487, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 35.9}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35887003131211, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.68}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.714067079126835, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42013196460903, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.73}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.33454195410013, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 35.64}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15865802951157, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.46}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.05833000317216, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 36.18}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.80384594574571, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.89}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.704387145116925, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 35.87}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.390201119706035, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.67}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51665393449366, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 35.76}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47215215116739, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.8}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23736100457609, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 35.46}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05058712512255, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.4}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6818558908999, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 35.98}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41750190965831, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.75}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.481794064864516, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.73}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30482002720237, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.66}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.929578848183155, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 36.12}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.62770397588611, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.3920029271394, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 35.63}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14828798547387, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.5}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.819047855213284, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 36.05}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75407513603568, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.02}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.52751501835883, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 35.87}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.509972928091884, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.78}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.488622987642884, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 35.78}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45463187806308, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.72}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15074893645942, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13249904103577, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.42}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.49466401152313, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 35.79}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27120906114578, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.56}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.87096792459488, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 36.1}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20880905166268, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.51}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.458554074168205, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 35.74}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28797985240817, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.56}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.33847213536501, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16833796352148, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.4}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.62128616310656, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 35.89}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37602103501558, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.626525085419416, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 35.88}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31860010884702, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.64200599491596, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 35.84}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36227983608842, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.58}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.20172999985516, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 35.49}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61504404246807, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.97}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35280193015933, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 35.6}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10500802844763, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.41}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.60924509353936, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 35.87}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.190100161358714, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 36.91}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.37616206146777, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.414581049233675, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.65}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.15926006063819, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43279096670449, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.7}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.101378828287125, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 35.35}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26288001053035, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.49}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.607246009632945, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.9}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32368000783026, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.63}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.42518292367458, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 35.61}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17890894040465, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.45}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.265581941232085, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.56}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32330002821982, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.62}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.6166051030159, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 35.75}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21982892602682, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.55}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.41102309525013, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 35.73}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33470989391208, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.84435794316232, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 36.0}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219559075310826, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.44}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32687181420624, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 35.5}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29221993125975, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.63}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.43747288919985, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 35.64}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.451210897415876, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.77}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.7965370900929, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 35.95}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34256117604673, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.66}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.71949599124491, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 35.88}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.457001861184835, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.60106482170522, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.8}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.585144164040685, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.67}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.653355091810226, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.86}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50413190573454, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.54144411161542, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 35.81}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45772200636566, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.54635497555137, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 35.82}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19893889501691, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.56}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35792210884392, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 35.59}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.427420960739255, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.67}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.68396603502333, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 35.95}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45194198563695, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65468502044678, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 35.83}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.262788973748684, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.73790707439184, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.87}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.308130180463195, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.62}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.29981107264757, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 35.61}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40749089047313, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.69}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.701536133885384, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 35.87}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67247498035431, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.83}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.2810418959707, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 35.62}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.445461843162775, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.74}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.528054021298885, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 35.7}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.145528011024, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.47}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.326611975207925, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 35.59}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.29776991531253, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.51731400936842, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 35.79}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366000935435295, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.31434203311801, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 35.6}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.925781078636646, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.26}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.838288048282266, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 35.96}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.506932159885764, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.69}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.561325054615736, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 35.69}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.583003053441644, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.86}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35709183476865, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 35.66}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36696997657418, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.64}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.32290298305452, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 35.62}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.378090899437666, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.62}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.91890915110707, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 36.16}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.506822030991316, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.77}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.395293056964874, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 35.71}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39539091475308, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.57}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.40112290345132, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 35.64}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61243401095271, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.71}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.65101491101086, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 35.73}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34878217801452, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.430113112553954, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 35.69}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41500107571483, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.72}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.5201838798821, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 35.7}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.296750117093325, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.61}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.323771907016635, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.56}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40347200073302, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.75}], "seconds": 21.3, "failures": 0}}, "finished_utc": "2026-09-30T16:34:35Z"} |