vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a262477a61c6fc68726abf87ee7c448b1dee4a1b
esh-pfi-infrastructure/services/intern-decision-serve/acceptance/gpu3-2026-09-30/r2/sets.json
T
vh a262477a61 feat(intern-decision): stack, DNS and GPU 3 acceptance for the SemIf replacement 
stacks/intern-decision: compose (GPU 1, :8033, hard VRAM cap as the single .env knob,
healthcheck, Homepage group 'AI - Eval & Retrieval'), .env.example and README.
dns: intern-decision.fv.internal -> fv-ml1 (synced to ana/esh/nh3).
acceptance on fv-ml1 GPU 3, 3 fresh processes: bit-identical to the Jev bench's native rows
(pooled 240/259, Wyrd 79/84, 0/560 flips, Δp 0), negative control 10/122/14, 0 flips across
restarts; largest accepted request 200 at a 10,134 MiB card peak under a 9.25 GiB cap; 503 and
recovery proven at a tight cap. GPU 1 deploy held: nvidia-smi Free on GPU 1 is 15,442 MiB.
2026-09-30 09:38:00 -07:00

1 line
703 KiB
JSON
Raw Blame History

{"label": "r2", "backend": "semif", "url": "http://127.0.0.1:18033", "started_utc": "2026-09-30T16:26:37Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.44668398052454, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 38.12}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.255457205697894, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.58}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.109945038333535, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.55}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.041146075353026, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 35.06}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.23380698263645, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 34.46}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.057333927601576, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.49}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97197581641376, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.68}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30079997330904, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.72}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.89894701912999, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.77}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.564093014225364, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.73}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42346097715199, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.52}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.418731885030866, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 35.58}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4688720330596, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.58}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.6795301977545, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 41.77}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.102279184386134, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 35.2}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.841886980459094, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.94}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.684354931116104, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.74}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61470317840576, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.79}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23300900310278, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.43}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.84897597506642, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.93}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.062718929722905, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 35.21}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.452701944857836, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 34.57}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.88074618205428, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 35.0}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92958799749613, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 35.05}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60902397334576, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 34.74}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.752744974568486, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.88}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.490472896024585, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.65}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66804398037493, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.76}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33147005550563, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.47}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82236692495644, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.0}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58252295665443, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.75}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63361391611397, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.76}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.57617203518748, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 34.73}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.672403967007995, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.78}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25764015689492, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.4}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59895311668515, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.74}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.300289841368794, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.46}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31851000338793, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.45}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32272982597351, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 34.46}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48894203454256, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.57}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.293039029464126, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 34.37}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.315950030460954, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.47}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.234978983178735, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.38}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44298010133207, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.54}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.45967103168368, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 34.49}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40479098632932, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.55}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13702689856291, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 34.33}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59655309841037, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.72}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.344638956710696, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.51}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47712215408683, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.52}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14432683773339, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.36}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56458195857704, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.72}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33831108547747, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.42}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.60275291278958, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.7}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3884211294353, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.51}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.54282207228243, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.64}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.461671978235245, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.58}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49944199807942, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.57}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22184803336859, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.41}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49661287292838, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.64}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26034890860319, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 34.37}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.241797894239426, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.4}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.064466858282685, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.24}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36752993427217, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.55}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14832803234458, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.32}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37176116555929, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.53}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15838794596493, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.31}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.709065014496446, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.84}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.140268133953214, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.28}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.402970949187875, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.6}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.136767990887165, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.31}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2613191138953, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.44}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.276949966326356, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.48}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.504761945456266, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.65}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.104306975379586, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 34.29}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.574122892692685, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.71}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.558474043384194, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 35.71}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75359503924847, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.79}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21709898672998, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.35}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82551595941186, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.82}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34837006777525, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.48}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49870206229389, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.61}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.938305012881756, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.08}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.715474143624306, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.92}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47463086619973, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.59}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.618294125422835, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.79}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23594802431762, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.4}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.53924183547497, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.7}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22117794677615, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.41}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.705104099586606, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.84}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96622606366873, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.18}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 43.97163190878928, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 41.08}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.766474997624755, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.88}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.513872142881155, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.68}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.347269942983985, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 34.49}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79934602789581, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.89}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3603911139071, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.57}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74931514635682, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.89}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33156993985176, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.51}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.599793169647455, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 34.76}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59808302856982, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.75}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67648502252996, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.76}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66951407305896, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.76}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86172717809677, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 35.01}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.759405095130205, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.91}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.35512890852988, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.41}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71241202391684, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 33.89}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.59118605032563, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 35.98}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.277749041095376, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.66}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.94523794203997, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 35.26}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.200227146968246, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.59}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.48643305152655, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.79}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96565609425306, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.41}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.40922105498612, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.8}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22486901097, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.47}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.399559980258346, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.78}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.018266040831804, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.39}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38428093492985, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.63}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.28939692862332, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.55}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.671971902251244, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.24}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11870708502829, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.16}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.286829901859164, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.25}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74763301387429, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 33.92}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.240659119561315, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.37}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94587596692145, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.05}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.066776072606444, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.27}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.676561925560236, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 33.86}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91032505594194, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.1}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.54568991623819, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 33.73}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03512600623071, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.2}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62411100231111, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 33.82}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22025919705629, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.39}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80411400273442, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 33.98}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87799395993352, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.02}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97245521470904, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.12}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04882599413395, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.17}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03352599404752, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.16}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.04973589628935, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.24}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19552792608738, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.33}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20606816932559, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.25}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01919503509998, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.09}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23724908195436, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.29}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.040065973997116, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.01}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.175887962803245, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.18}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.032806780189276, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 34.02}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.464231019839644, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 34.52}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25660894997418, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 34.29}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28960920125246, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 34.26}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.14469796977937, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.24}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.456660997122526, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 34.52}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.822943948209286, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 33.87}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.4224609695375, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 34.49}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.151167169213295, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 34.38}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05082694068551, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 34.26}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.054846063256264, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 34.25}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24084887653589, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 34.53}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23278781399131, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 34.46}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949565168470144, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 34.17}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90587496384978, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 33.96}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16240799985826, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 34.46}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53792920522392, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 33.75}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05193684436381, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 34.19}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.715112160891294, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 33.91}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91852488555014, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 34.21}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.991525907069445, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 34.04}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03861706890166, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 34.17}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.492559127509594, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 33.66}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24692901596427, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 34.45}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01126598753035, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 34.18}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.041685776785016, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 34.23}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30337996967137, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 34.58}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.10791198723018, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 36.24}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17223787680268, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 34.34}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.467902060598135, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 34.74}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85023309662938, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 33.98}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.229899084195495, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 34.26}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9270951487124, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 34.06}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85475396923721, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 34.1}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03950601629913, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 34.26}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46401215903461, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 34.62}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.678232019767165, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 33.91}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.82357398793101, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 34.06}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.560420179739594, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 33.81}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13636798784137, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 34.39}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65757109411061, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 33.97}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0757170021534, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 34.28}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.037295987829566, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 34.3}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.973054986447096, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 34.17}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12979797273874, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 34.28}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.153696874156594, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 34.44}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.977285984903574, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 34.18}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.103047128766775, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 34.36}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99741582386196, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 34.29}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.187947891652584, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 34.37}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05677599646151, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 34.35}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13394794613123, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 34.49}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.61223012022674, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 33.81}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61716396547854, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 34.83}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.766533041372895, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 34.0}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42686007171869, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 34.69}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.841783905401826, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.12}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.059046095237136, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 34.37}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78101394325495, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 34.04}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69521391950548, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 34.95}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62512102164328, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 33.75}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01238613575697, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 34.31}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.55640012584627, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 33.84}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.884255008772016, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 34.02}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67792188934982, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 33.82}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17468795366585, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 34.44}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.903555039316416, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.09}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8710239417851, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 34.1}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06657700240612, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 34.22}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.141327979043126, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 34.37}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59104090183973, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 33.85}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37713093869388, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 34.57}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9736950378865, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 34.22}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41724090650678, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 34.7}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39163000136614, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 34.5}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.178847938776016, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 34.4}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06358699128032, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 34.28}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.701164139434695, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 34.96}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09447709843516, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 34.3}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05421695485711, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 34.22}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.594703026115894, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 34.8}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.429180927574635, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 34.59}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.037425907328725, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 34.34}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.6131241209805, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 34.91}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.063237046822906, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 34.32}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.28917986154556, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 34.58}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.156777922064066, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 34.44}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.176007870584726, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 34.29}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79815400391817, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 34.01}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26775920949876, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 34.57}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.48185892961919, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 33.75}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01904695481062, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 34.16}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83579410426319, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 34.07}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.314829882234335, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 34.52}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.620229948312044, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 33.82}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17690892517567, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 34.43}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.650391994044185, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 33.99}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17411798425019, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 34.38}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.692542023956776, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 33.89}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27822890505195, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 34.42}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.627341993153095, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 33.89}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.280819145962596, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.38}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75903403200209, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 34.02}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.4424160476774, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 35.39}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.247860968112946, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 34.4}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.975396098569036, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.1}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.820653825998306, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 34.09}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26641810499132, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.47}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.309879902750254, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 34.53}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11046697571874, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.21}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.147146882489324, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 34.32}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00666595250368, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.11}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.46000095270574, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 34.64}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15690807439387, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.745243007317185, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 33.86}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.208799039945006, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.123706890270114, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 34.36}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83673404157162, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.07}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20814804546535, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 34.33}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9010039139539, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.09}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.494962103664875, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00039582327008, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.198618054389954, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96828614920378, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.26}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33603982254863, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07188693806529, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.31}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.789454916492105, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.024616030976176, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39601094275713, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 34.63}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01258497312665, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.34}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87881398946047, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 34.11}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47443202883005, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.67}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3000199906528, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 34.59}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98909608647227, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.24}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3727201949805, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.957825999706984, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.13}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184017011895776, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 34.44}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94300493225455, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.17}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27197996340692, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.226418033242226, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2073189355433, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 34.47}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.059335969388485, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.35}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.765356013551354, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 34.94}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10658801719546, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.83852490596473, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 35.14}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.205108907073736, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.422270979732275, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 35.11}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.08454414643347, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.34}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 35.91087204404175, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 34.31}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8973040021956, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.23}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1749890036881, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 34.45}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10568696260452, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.4}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41049114614725, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 34.63}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219329038634896, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.37}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42708987556398, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 34.7}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.258289055898786, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.6}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34290017746389, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 34.66}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69366193935275, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.91}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22388902679086, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.979005904868245, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.24}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1631879825145, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.949384957551956, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.1}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32484998181462, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 34.61}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11949707940221, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21609804779291, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 34.46}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30309009552002, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.49}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2936490457505, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 34.62}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01664600521326, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.28}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17129793949425, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 34.5}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06913720816374, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.38}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.220368860289454, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 34.39}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.902404157444835, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.16}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11383696645498, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 34.41}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.62745212204754, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.89}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.67316392622888, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 34.81}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.784562980756164, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 33.91}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18891809694469, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 34.3}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.266249069944024, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.36}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.27248916402459, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 34.41}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49627200886607, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.71}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.81487403810024, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 33.95}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01546601951122, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.184678949415684, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 34.27}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80344391614199, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.93}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03377698548138, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 34.28}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848853807896376, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.1}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.529348930343986, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 33.82}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56332101672888, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79147292859852, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79122403264046, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.0}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88425407744944, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84158390387893, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.1}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71621298417449, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 33.89}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.785762989893556, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.05}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92184598185122, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95285506546497, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.927014822140336, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.848923889920115, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.0}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.66127077303827, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 33.86}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95705602876842, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.18}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.627830006182194, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 33.91}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57245100475848, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.83}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.90529405139387, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 34.2}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94750485010445, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.23}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70207201503217, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95761505514383, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.23}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84633388184011, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 34.04}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88773396424949, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69330198317766, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 33.94}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.68989217840135, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75332199782133, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 33.94}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67185199446976, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.01}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.927883978933096, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.083216942846775, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.14}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.219209130853415, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 34.42}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.193749099969864, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.4}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15861798264086, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 34.09}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02912596054375, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.12}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99467587284744, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 34.11}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.024846067652106, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07614680752158, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 34.28}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06888691522181, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.32}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06672601401806, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.308290135115385, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0520269498229, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 34.38}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15148800984025, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.27}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.9738859590143, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 34.08}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31427900493145, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.52}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93277505226433, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 34.19}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.197297904640436, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.16}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99111589230597, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.249339977279305, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.57}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.377950036898255, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 34.53}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.064017029479146, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.33}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11127699352801, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 34.38}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.815223982557654, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.03}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.865063942968845, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.103266920894384, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.37}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15607803314924, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 34.37}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67656099423766, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.95}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0293369051069, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 34.3}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98464599438012, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.34}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15182701125741, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 34.31}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.77299292758107, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 33.92}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97705501690507, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 34.25}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08778694272041, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.39}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91980498842895, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 34.15}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34738007187843, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.36}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.924261858686805, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 37.02}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95264412090182, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.27}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28411200083792, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 35.37}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12979797273874, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.628632970154285, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 34.75}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88197489827871, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.05}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.768535083159804, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.32177009806037, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.57}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.92222682386637, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 35.14}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91294486634433, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.99}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.95208805240691, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02805587090552, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.25}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.74021496064961, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.197898142039776, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.49}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.86863596178591, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 35.02}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97245614603162, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.17}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.092120084911585, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 35.33}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.118196953088045, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.35970303043723, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 35.57}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19504806213081, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.63027302920818, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13502804748714, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.22}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75556501932442, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 34.88}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33367007225752, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.64}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.09155896306038, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 35.38}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.550612119957805, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.67}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.97404794022441, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 35.1}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37356001511216, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.59}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.52358304336667, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 34.7}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25181892514229, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.51}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.108499022200704, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 35.4}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26628888398409, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.54}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.988678086549044, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 35.24}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.70997195877135, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 33.97}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.842626217752695, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 35.0}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.85002401471138, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.06}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.69075404852629, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.71159199438989, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.01}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.581582786515355, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 34.78}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05278690904379, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.29}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.831834983080626, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 35.06}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1521080378443, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.37}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.99200803041458, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 35.29}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26338897831738, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.764016073197126, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 34.91}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10448695346713, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.39}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.12199993990362, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 35.24}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.142917048186064, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.44}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08706998825073, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 35.28}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92029579542577, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.26}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.770704831928015, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 34.95}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06533694639802, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.31}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.72488492541015, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 34.95}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.109137047082186, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.32}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.10638911090791, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 35.29}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2432479634881, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.39}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01732882857323, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.2}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.212077993899584, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.41}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.80165477655828, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 34.9}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18247916549444, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.45}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79603494331241, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10946696810424, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.42}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.316902006044984, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 35.49}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.356629967689514, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 34.58}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7611662261188, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01378614641726, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.24}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.682224065065384, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00655605643988, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.33}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.77555492706597, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.0}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.73664294183254, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 33.99}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.68605412915349, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 34.9}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.720452131703496, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 33.97}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.727355025708675, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 34.96}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89654404297471, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.06}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44949190877378, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 34.69}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.88608482480049, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.09}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56884206086397, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 34.72}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.933585070073605, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.28}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 51.05949891731143, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 40.58}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.86027913540602, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.99}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.907259004190564, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 36.09}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 40.44921789318323, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 37.53}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 39.60481798276305, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 36.69}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.61141304858029, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.50120515562594, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 35.66}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.01093902438879, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.33}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.66063617169857, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 35.84}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.927808007225394, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 35.17}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.34659280255437, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 35.54}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.64046402648091, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08962902985513, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.42}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33282000757754, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.6}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08367904275656, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 35.28}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31854003854096, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.52}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.00690802745521, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 35.21}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.30955999344587, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.54}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.83918614499271, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 35.01}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.026925943791866, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.26}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.99337800592184, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 35.18}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.368229823186994, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.53}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.13883988186717, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.34}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98440594598651, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.27}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90758713148534, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 35.13}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08416689187288, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.32}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.75140503421426, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.92890494130552, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.19}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.00300811417401, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 35.1}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.69216204434633, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 33.89}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50622202642262, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 34.64}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03443589620292, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.15}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.93478710576892, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.111018085852265, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.25}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.08818897232413, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 35.18}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.58095297962427, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.67}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.36012305691838, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 35.46}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34349994920194, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.42}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.27393217943609, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 35.37}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37106989137828, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.64}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.05428883060813, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 35.29}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.386061158031225, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.53}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.23584015481174, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 35.34}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.39719116128981, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.79989620670676, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 34.99}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.42543002590537, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.6}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.805835949257016, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 34.9}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.22162893973291, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.47}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.091839058324695, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 35.24}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25028899498284, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.51}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.114198949187994, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 35.25}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.227059016004205, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.55}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98735816963017, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.13}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83857386931777, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.16}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.90008695796132, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 35.13}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19667810946703, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.43}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.04708900861442, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 35.28}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.177417892962694, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.29}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.98843780532479, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 35.11}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23614918999374, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.33}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.05192792788148, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 35.08}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26732009090483, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.36}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.66590403392911, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 34.77}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19124780036509, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.25}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.28226192854345, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 35.38}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.55816305056214, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.68}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.928147008642554, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 34.93}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.34753094613552, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.5}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.087488850578666, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38199989311397, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.59}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.81934594735503, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 34.83}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.10978711023927, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.39}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.88401698693633, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 35.15}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25325898267329, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.49}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.7266148570925, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 34.98}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26948914118111, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.47}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 38.192749954760075, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 35.46}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15572785586119, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.32}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.883436074480414, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20280900597572, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.44}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.73043514229357, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 35.02}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36678999848664, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.45}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.76174504309893, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 34.99}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.017345894128084, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.23}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.609933177009225, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 34.71}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.98468510992825, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.17}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.36928990110755, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 34.53}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.135668098926544, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.38}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.82323584891856, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.01}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1592880692333, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.3}], "seconds": 21.0, "failures": 0}, "rotations": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.92347499914467, "probs": [0.021278230729761302, 0.041549605758459535, 0.9371721635117792], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.96534493379295, "probs": [0.9569751557868662, 0.022187022115204392, 0.020837822097929372], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42147886008024, "probs": [0.028764030986406904, 0.9453276881980787, 0.025908280815514436], "top": "supported", "agreement": 1.0, "input_tokens": 196}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.69498199038208, "probs": [0.9253638152569406, 0.02190751448970375, 0.052728670253355744], "top": "insufficient", "agreement": 1.0, "input_tokens": 196}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39131797850132, "probs": [0.9597742641946788, 0.021791404944545737, 0.01843433086077556], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22496607154608, "probs": [0.0761485956363141, 0.025667636008847486, 0.8981837683548385], "top": "contradicted", "agreement": 1.0, "input_tokens": 206}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.49095994792879, "probs": [0.9159676813214028, 0.0555726559991739, 0.028459662679423257], "top": "contradicted", "agreement": 1.0, "input_tokens": 208}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65389203839004, "probs": [0.02148731809580675, 0.05171731055352475, 0.9267953713506685], "top": "insufficient", "agreement": 1.0, "input_tokens": 200}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.65612185746431, "probs": [0.025695874936561842, 0.016912927133118746, 0.9573911979303193], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41266901418567, "probs": [0.9650011484904946, 0.014421131045601873, 0.02057772046390364], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.15835897624493, "probs": [0.9685134503822489, 0.011261347946432, 0.020225201671319158], "top": "supported", "agreement": 1.0, "input_tokens": 193}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39189912378788, "probs": [0.015601263068851802, 0.9601947660643307, 0.024203970866817515], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98396283946931, "probs": [0.008885594588144235, 0.029266695521859745, 0.9618477098899961], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22534605115652, "probs": [0.0467350628868348, 0.02343846611289458, 0.9298264710002706], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.68849183619022, "probs": [0.02661110718913202, 0.02251153411335893, 0.950877358697509], "top": "contradicted", "agreement": 1.0, "input_tokens": 199}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70519207976758, "probs": [0.9629993612353499, 0.018113485593456775, 0.01888715317119321], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24773597717285, "probs": [0.8820069055961203, 0.07635736188763995, 0.04163573251623969], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76509311795235, "probs": [0.02701681560472035, 0.9453954270569487, 0.027587757338330996], "top": "supported", "agreement": 1.0, "input_tokens": 198}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73087190277874, "probs": [0.02293683426823875, 0.9487912139983355, 0.028271951733425657], "top": "supported", "agreement": 1.0, "input_tokens": 194}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70189310237765, "probs": [0.019367602097265263, 0.013573064959569787, 0.9670593329431649], "top": "insufficient", "agreement": 1.0, "input_tokens": 194}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03357299603522, "probs": [0.03987925527749218, 0.02220468207692111, 0.9379160626455868], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.41814388893545, "probs": [0.02674495094579214, 0.9358820879397224, 0.03737296111448533], "top": "contradicted", "agreement": 1.0, "input_tokens": 211}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3368390314281, "probs": [0.03051840202788244, 0.9419931233693989, 0.027488474602718756], "top": "contradicted", "agreement": 1.0, "input_tokens": 201}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.7668428402394, "probs": [0.9573434428870992, 0.022664616232544885, 0.019991940880355972], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18864495679736, "probs": [0.20778523836492876, 0.6164398690397396, 0.17577489259533158], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.51035008393228, "probs": [0.01494019219967109, 0.958782768824845, 0.026277038975483776], "top": "permitted", "agreement": 1.0, "input_tokens": 218}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.67838209681213, "probs": [0.038717451842913995, 0.06953592628537188, 0.8917466218717142], "top": "permitted", "agreement": 1.0, "input_tokens": 203}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5801110342145, "probs": [0.8769652162510786, 0.06982844563770668, 0.053206338111214745], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18211591988802, "probs": [0.8731504724710493, 0.09124479732837286, 0.03560473020057783], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.5503198876977, "probs": [0.03149783270455631, 0.9131032971636024, 0.055398870131841214], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21073616109788, "probs": [0.1973515606103041, 0.4555434753670417, 0.3471049640226543], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 209}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13833607546985, "probs": [0.02274552846853418, 0.9023369803119639, 0.07491749121950192], "top": "insufficient", "agreement": 1.0, "input_tokens": 209}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13438587076962, "probs": [0.09302877372922651, 0.07088401170776323, 0.8360872145630103], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.49622913822532, "probs": [0.8635450750116728, 0.09024102515984218, 0.04621389982848493], "top": "permitted", "agreement": 1.0, "input_tokens": 211}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43088987469673, "probs": [0.028891977481104857, 0.0412263798327744, 0.9298816426861207], "top": "permitted", "agreement": 1.0, "input_tokens": 202}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8149931486696, "probs": [0.8174524855587436, 0.08365642576901129, 0.098891088672245], "top": "insufficient", "agreement": 1.0, "input_tokens": 204}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00847385264933, "probs": [0.017660819839882137, 0.04340570512192549, 0.9389334750381924], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32240709289908, "probs": [0.06132539226815876, 0.8373735739130129, 0.10130103381882835], "top": "prohibited", "agreement": 1.0, "input_tokens": 214}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.02946400083601, "probs": [0.887606329608409, 0.06778068214997746, 0.04461298824161357], "top": "prohibited", "agreement": 1.0, "input_tokens": 213}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16801499761641, "probs": [0.6913075067568434, 0.07074700515018907, 0.23794548809296753], "top": "insufficient", "agreement": 1.0, "input_tokens": 212}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3632078692317, "probs": [0.9055821252398959, 0.059736224410951576, 0.03468165034915258], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54104996845126, "probs": [0.034796330955504114, 0.9473839012637663, 0.017819767780729617], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00907409004867, "probs": [0.06676731835885638, 0.04041944853804973, 0.8928132331030938], "top": "permitted", "agreement": 1.0, "input_tokens": 208}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38725787773728, "probs": [0.0792938381036179, 0.09571489465194817, 0.8249912672444338], "top": "insufficient", "agreement": 1.0, "input_tokens": 215}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19659682177007, "probs": [0.08689479609139293, 0.6606481239315956, 0.2524570799770115], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8967539574951, "probs": [0.8171805101277473, 0.07074519344147624, 0.11207429643077639], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.55293108336627, "probs": [0.103178626971303, 0.7367489124270009, 0.16007246060169616], "top": "prohibited", "agreement": 1.0, "input_tokens": 209}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2475279904902, "probs": [0.08560085550345033, 0.3403340342335013, 0.5740651102630484], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.52908287011087, "probs": [0.017814439341391167, 0.9671155236327101, 0.015070037025898734], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.42857197672129, "probs": [0.9513649777837366, 0.01716163808274369, 0.03147338413351958], "top": "B", "agreement": 1.0, "input_tokens": 201}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.16081696934998, "probs": [0.025903351803450637, 0.9255875837884197, 0.048509064408129646], "top": "B", "agreement": 1.0, "input_tokens": 192}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.50336206890643, "probs": [0.019189428631759722, 0.9189139403005832, 0.061896631067657096], "top": "A", "agreement": 1.0, "input_tokens": 192}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02719499729574, "probs": [0.016185500037668298, 0.9553471934068705, 0.028467306555461265], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.10950808040798, "probs": [0.014798671264711157, 0.9697705732073082, 0.015430755527980513], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19649803265929, "probs": [0.025560818133635414, 0.022080003391839246, 0.9523591784745253], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08377703465521, "probs": [0.05311118840492001, 0.6811115463281155, 0.2657772652669644], "top": "A", "agreement": 1.0, "input_tokens": 190}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.83602606877685, "probs": [0.026144179410345973, 0.9539350561208116, 0.019920764468842493], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.79188400879502, "probs": [0.9592258084083156, 0.024179546083237823, 0.016594645508446512], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.38954117894173, "probs": [0.015353470774838725, 0.9649135180343748, 0.019733011190786628], "top": "B", "agreement": 1.0, "input_tokens": 197}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.29839007742703, "probs": [0.6229250573303873, 0.29341037282098503, 0.08366456984862763], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.43460003100336, "probs": [0.9523442934756741, 0.028977529107373938, 0.018678177416952112], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.87978409603238, "probs": [0.9509486773226812, 0.026062332698517814, 0.022988989978801007], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11371586471796, "probs": [0.028605829867980576, 0.9599959678670565, 0.01139820226496276], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.58347101323307, "probs": [0.027524206942156276, 0.10494883289705875, 0.8675269601607849], "top": "insufficient", "agreement": 1.0, "input_tokens": 192}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08665489032865, "probs": [0.025763001405599038, 0.9598922160922182, 0.014344782502182778], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.93993519432843, "probs": [0.01775221293583573, 0.018510449440082843, 0.9637373376240814], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2998401466757, "probs": [0.05805098351023624, 0.04331699912016453, 0.8986320173695993], "top": "B", "agreement": 1.0, "input_tokens": 200}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.24862788245082, "probs": [0.40713917224854596, 0.04723599385140831, 0.5456248339000457], "top": "A", "agreement": 1.0, "input_tokens": 193}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1625680886209, "probs": [0.9628824551152317, 0.020107611199832023, 0.017009933684936158], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90692493133247, "probs": [0.017791047056264133, 0.965845589126288, 0.016363363817447876], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.35855002328753, "probs": [0.9697536223339248, 0.01200584747578968, 0.018240530190285528], "top": "A", "agreement": 1.0, "input_tokens": 194}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56981309875846, "probs": [0.46166613268095874, 0.4246186674610029, 0.11371519985803841], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 192}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.63864389061928, "probs": [0.9434733126902809, 0.03465269924248234, 0.021873988067236654], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.29545990377665, "probs": [0.05653147014528852, 0.049865123655206005, 0.8936034061995054], "top": "supported", "agreement": 1.0, "input_tokens": 214}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1862378846854, "probs": [0.9646005420197569, 0.019726610085808626, 0.01567284789443457], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70744192227721, "probs": [0.011200303179937986, 0.9433281975088483, 0.045471499311213706], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.3328399322927, "probs": [0.04435070832583449, 0.8823880770841831, 0.07326121458998254], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22806893661618, "probs": [0.38436219601806726, 0.39248484865749256, 0.22315295532444018], "top": "prohibited", "agreement": 0.3333333333333333, "input_tokens": 229}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.13498720340431, "probs": [0.038671196720481874, 0.051824945670009545, 0.9095038576095087], "top": "permitted", "agreement": 1.0, "input_tokens": 222}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.56370292417705, "probs": [0.9386225473366789, 0.024670771221418437, 0.03670668144190261], "top": "insufficient", "agreement": 1.0, "input_tokens": 221}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2614789698273, "probs": [0.06338502047709155, 0.20445210704639485, 0.7321628724765136], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1336169950664, "probs": [0.01023906219590939, 0.9776572412337166, 0.012103696570374124], "top": "A", "agreement": 1.0, "input_tokens": 210}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22118902392685, "probs": [0.08343895995982653, 0.048442981731284813, 0.8681180583088886], "top": "A", "agreement": 1.0, "input_tokens": 209}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.68476212210953, "probs": [0.028590681320234453, 0.07026846391461965, 0.9011408547651459], "top": "insufficient", "agreement": 1.0, "input_tokens": 211}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27347999252379, "probs": [0.9485370017696566, 0.021991387483579136, 0.02947161074676416], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57483316026628, "probs": [0.1476984273576982, 0.06533845399896182, 0.78696311864334], "top": "supported", "agreement": 1.0, "input_tokens": 227}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.47177207283676, "probs": [0.9659641119707486, 0.01300233393992522, 0.021033554089326202], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.55744280666113, "probs": [0.9452042665175415, 0.037745228086007734, 0.01705050539645064], "top": "insufficient", "agreement": 1.0, "input_tokens": 202}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.91050493530929, "probs": [0.7928775526060625, 0.1207272484983066, 0.08639519889563085], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27028997987509, "probs": [0.4902958520500278, 0.23093932363732236, 0.2787648243126498], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 231}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.11481708474457, "probs": [0.08132590401657058, 0.05465960600149562, 0.8640144899819338], "top": "permitted", "agreement": 1.0, "input_tokens": 216}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22538882307708, "probs": [0.2290171921944468, 0.6937872957432885, 0.07719551206226469], "top": "insufficient", "agreement": 1.0, "input_tokens": 223}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.17888811975718, "probs": [0.030141465776599607, 0.9500195362761745, 0.019838997947225906], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90977501124144, "probs": [0.025305433525381633, 0.03185065213808263, 0.9428439143365358], "top": "B", "agreement": 1.0, "input_tokens": 222}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.25929781235754, "probs": [0.05059101952456338, 0.04279722164027021, 0.9066117588351665], "top": "B", "agreement": 1.0, "input_tokens": 215}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.13965801894665, "probs": [0.012768350746295443, 0.01860437029559649, 0.9686272789581081], "top": "insufficient", "agreement": 1.0, "input_tokens": 201}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.95979611389339, "probs": [0.9208322432392101, 0.04082517423414474, 0.038342582526645214], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90040381066501, "probs": [0.8434852492825246, 0.05011598360172626, 0.10639876711574907], "top": "supported", "agreement": 1.0, "input_tokens": 219}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73847289197147, "probs": [0.023619578875477214, 0.01956735197412583, 0.9568130691503969], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27584701776505, "probs": [0.8650697558641954, 0.10248576421341996, 0.03244447992238454], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.56231089867651, "probs": [0.10549476831325036, 0.0754944842886127, 0.819010747398137], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38530792109668, "probs": [0.335350091795104, 0.11069809870287566, 0.5539518095020204], "top": "prohibited", "agreement": 1.0, "input_tokens": 221}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3536687977612, "probs": [0.8825022468959398, 0.07026933060840494, 0.047228422495655314], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01221404410899, "probs": [0.06932232571305925, 0.9077945737563922, 0.022883100530548473], "top": "insufficient", "agreement": 1.0, "input_tokens": 220}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.72152305394411, "probs": [0.009281733826954271, 0.9839356748468795, 0.006782591326166279], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18155503086746, "probs": [0.008340312669325892, 0.9815921692199244, 0.010067518110749797], "top": "B", "agreement": 1.0, "input_tokens": 230}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.5738529432565, "probs": [0.012023447059675983, 0.9711751830662149, 0.016801369874109195], "top": "B", "agreement": 1.0, "input_tokens": 223}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.15951504930854, "probs": [0.011643930956106696, 0.9806920781093156, 0.007663990934577689], "top": "insufficient", "agreement": 1.0, "input_tokens": 210}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.76964581944048, "probs": [0.900478485395825, 0.023178368052708644, 0.07634314655146633], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.36706796847284, "probs": [0.06499692312355161, 0.047496249800495424, 0.887506827075953], "top": "supported", "agreement": 1.0, "input_tokens": 224}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.81810399889946, "probs": [0.3922836361905562, 0.48352898160684454, 0.12418738220259931], "top": "insufficient", "agreement": 0.3333333333333333, "input_tokens": 220}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90648395009339, "probs": [0.0560499070635644, 0.020116182060737196, 0.9238339108756985], "top": "insufficient", "agreement": 1.0, "input_tokens": 208}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.863933917135, "probs": [0.17500642472273137, 0.21124871005668264, 0.613744865220586], "top": "permitted", "agreement": 1.0, "input_tokens": 227}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.85776709578931, "probs": [0.5914884424684399, 0.035143500631624254, 0.37336805689993585], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 228}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.78900296241045, "probs": [0.9046569284481161, 0.042704941598692904, 0.05263812995319102], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13665620237589, "probs": [0.029736313438628644, 0.033014009977779125, 0.9372496765835922], "top": "insufficient", "agreement": 1.0, "input_tokens": 218}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.82867288030684, "probs": [0.7432028429041843, 0.03582480375679601, 0.2209723533390198], "top": "A", "agreement": 1.0, "input_tokens": 221}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.19941593520343, "probs": [0.06632110060786917, 0.8329169964218126, 0.10076190297031828], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.72478221729398, "probs": [0.7397871533568318, 0.1860713523015022, 0.07414149434166593], "top": "A", "agreement": 1.0, "input_tokens": 207}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.02457611821592, "probs": [0.9248773731002589, 0.014113602375072627, 0.06100902452466855], "top": "insufficient", "agreement": 1.0, "input_tokens": 197}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.99382594227791, "probs": [0.9556486965100951, 0.021248690458660385, 0.02310261303124464], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.0531169641763, "probs": [0.010853780626859795, 0.015814708468139885, 0.9733315109050004], "top": "supported", "agreement": 1.0, "input_tokens": 212}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.62550090253353, "probs": [0.9606317563420671, 0.018450800471885003, 0.020917443186047876], "top": "supported", "agreement": 1.0, "input_tokens": 202}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.27171001397073, "probs": [0.8027819128710822, 0.011505476315571077, 0.18571261081334672], "top": "contradicted", "agreement": 0.6666666666666666, "input_tokens": 203}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.97806291095912, "probs": [0.8487397319059693, 0.05482797840153974, 0.09643228969249089], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.17209901101887, "probs": [0.019700876806489794, 0.943405397557902, 0.036893725635608235], "top": "permitted", "agreement": 1.0, "input_tokens": 221}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07685504853725, "probs": [0.9481975719388965, 0.021083015776241384, 0.030719412284862045], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24921701289713, "probs": [0.11594495476456004, 0.2790651305948641, 0.6049899146405758], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.80278101749718, "probs": [0.011620895003629282, 0.009627189199890869, 0.9787519157964798], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03300419077277, "probs": [0.9712341256474011, 0.015134214927072205, 0.013631659425526738], "top": "B", "agreement": 1.0, "input_tokens": 212}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17468606121838, "probs": [0.9748621568921099, 0.014266993806726613, 0.010870849301163494], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25843594223261, "probs": [0.01218702517662872, 0.9640155969735945, 0.02379737784977667], "top": "insufficient", "agreement": 1.0, "input_tokens": 191}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.07350508123636, "probs": [0.05216159441960384, 0.25033305566222464, 0.6975053499181716], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98382314108312, "probs": [0.0744428867114499, 0.8598927356841719, 0.06566437760437807], "top": "supported", "agreement": 1.0, "input_tokens": 221}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9482929520309, "probs": [0.031673836031138235, 0.010902017412968387, 0.9574241465558934], "top": "supported", "agreement": 1.0, "input_tokens": 229}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21386587060988, "probs": [0.9598638711537073, 0.029206330829344665, 0.01092979801694802], "top": "insufficient", "agreement": 1.0, "input_tokens": 226}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12577602639794, "probs": [0.3126088342455135, 0.527299092068065, 0.1600920736864215], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11021502315998, "probs": [0.018596715235427216, 0.8540525288330012, 0.1273507559315715], "top": "permitted", "agreement": 1.0, "input_tokens": 215}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.35944905132055, "probs": [0.2682095319185024, 0.5694223884119577, 0.16236807966953995], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.05736386403441, "probs": [0.5197715159798874, 0.040530340274099394, 0.4396981437460133], "top": "insufficient", "agreement": 0.6666666666666666, "input_tokens": 224}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.28141702525318, "probs": [0.03677949530401433, 0.9404844838908847, 0.02273602080510092], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.19254400767386, "probs": [0.09722552658518739, 0.7089119171844285, 0.19386255623038412], "top": "B", "agreement": 1.0, "input_tokens": 218}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.53492093458772, "probs": [0.0446355264117619, 0.029378934066784906, 0.9259855395214532], "top": "B", "agreement": 1.0, "input_tokens": 209}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.34934792667627, "probs": [0.8956127457317613, 0.023540301062398936, 0.08084695320583983], "top": "insufficient", "agreement": 1.0, "input_tokens": 207}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8773051481694, "probs": [0.9215420659013949, 0.05709241087032677, 0.021365523228278453], "top": "contradicted", "agreement": 1.0, "input_tokens": 204}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.38078797981143, "probs": [0.024650674719169527, 0.05690076079826144, 0.9184485644825692], "top": "contradicted", "agreement": 1.0, "input_tokens": 215}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1340859849006, "probs": [0.017881542110937673, 0.0314503320010902, 0.9506681258879721], "top": "contradicted", "agreement": 1.0, "input_tokens": 236}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84610195271671, "probs": [0.015971281239954956, 0.021403804942628207, 0.9626249138174169], "top": "supported", "agreement": 1.0, "input_tokens": 209}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01892399042845, "probs": [0.9469841068616602, 0.027062216114961006, 0.025953677023378714], "top": "supported", "agreement": 1.0, "input_tokens": 220}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.79540098458529, "probs": [0.9655262695437641, 0.017417092730978554, 0.01705663772525738], "top": "supported", "agreement": 1.0, "input_tokens": 241}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.00976420007646, "probs": [0.9523229876734454, 0.019887100792948465, 0.02778991153360623], "top": "contradicted", "agreement": 1.0, "input_tokens": 194}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9653619993478, "probs": [0.02944395390697739, 0.022909170093210977, 0.9476468759998116], "top": "contradicted", "agreement": 1.0, "input_tokens": 205}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1517860032618, "probs": [0.028467305657075486, 0.01618550022217239, 0.9553471941207522], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.62260898202658, "probs": [0.9626180927581529, 0.027508993151357353, 0.009872914090489732], "top": "supported", "agreement": 1.0, "input_tokens": 205}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.85486197285354, "probs": [0.01216177373585067, 0.025820060282101747, 0.9620181659820476], "top": "supported", "agreement": 1.0, "input_tokens": 216}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.75384001620114, "probs": [0.008947931903485177, 0.02245643770922697, 0.9685956303872878], "top": "supported", "agreement": 1.0, "input_tokens": 237}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.21442606113851, "probs": [0.03490400868942244, 0.07256938070358598, 0.8925266106069916], "top": "contradicted", "agreement": 1.0, "input_tokens": 198}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30745703727007, "probs": [0.8560321676779682, 0.10355825471147521, 0.04040957761055657], "top": "contradicted", "agreement": 1.0, "input_tokens": 209}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.01811397261918, "probs": [0.9275028632254695, 0.04760344873379892, 0.024893688040731596], "top": "contradicted", "agreement": 1.0, "input_tokens": 230}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.9652430228889, "probs": [0.9261949727483907, 0.021028996331540862, 0.0527760309200684], "top": "supported", "agreement": 1.0, "input_tokens": 211}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.06168496794999, "probs": [0.03674381458625419, 0.02368412668428037, 0.9395720587294654], "top": "supported", "agreement": 1.0, "input_tokens": 222}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12497485615313, "probs": [0.026670715956627202, 0.020321964093249996, 0.9530073199501229], "top": "supported", "agreement": 1.0, "input_tokens": 243}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.76580301858485, "probs": [0.08330838841015804, 0.7031959025468014, 0.21349570904304058], "top": "prohibited", "agreement": 1.0, "input_tokens": 212}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.48308002762496, "probs": [0.18172827933132565, 0.6645395854888777, 0.15373213517979664], "top": "prohibited", "agreement": 1.0, "input_tokens": 223}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.40966899134219, "probs": [0.08895150060315864, 0.816337598890086, 0.09471090050675535], "top": "prohibited", "agreement": 1.0, "input_tokens": 244}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.29607720673084, "probs": [0.0293692258945804, 0.11923451516955454, 0.851396258935865], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4426790215075, "probs": [0.7559638022373625, 0.22951644325448917, 0.014519754508148302], "top": "permitted", "agreement": 1.0, "input_tokens": 220}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.54285999387503, "probs": [0.9180643670041458, 0.055699863652112935, 0.026235769343741142], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.88290495797992, "probs": [0.8545877320672847, 0.07877388866940015, 0.06663837926331509], "top": "prohibited", "agreement": 1.0, "input_tokens": 208}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.88860395364463, "probs": [0.10813821274832802, 0.06970313933726799, 0.822158647914404], "top": "prohibited", "agreement": 1.0, "input_tokens": 219}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.23563919216394, "probs": [0.04460783183966404, 0.029981179056752127, 0.9254109891035839], "top": "prohibited", "agreement": 1.0, "input_tokens": 240}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.89105496183038, "probs": [0.9383472668786358, 0.03989758917731366, 0.021755143944050585], "top": "permitted", "agreement": 1.0, "input_tokens": 214}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42865889146924, "probs": [0.019454336721104887, 0.06822606373845527, 0.9123195995404397], "top": "permitted", "agreement": 1.0, "input_tokens": 225}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.75674404390156, "probs": [0.015551699476841353, 0.0471124973010445, 0.9373358032221141], "top": "permitted", "agreement": 1.0, "input_tokens": 246}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4506590589881, "probs": [0.0377743579001357, 0.05503984914430352, 0.9071857929555608], "top": "prohibited", "agreement": 1.0, "input_tokens": 216}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.15061805583537, "probs": [0.9057645436591656, 0.057300812015498664, 0.03693464432533565], "top": "prohibited", "agreement": 1.0, "input_tokens": 227}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30558787286282, "probs": [0.9388430475426657, 0.037491202062820105, 0.023665750394514205], "top": "prohibited", "agreement": 1.0, "input_tokens": 248}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.08859506808221, "probs": [0.28824200219090185, 0.5868851532425197, 0.12487284456657842], "top": "permitted", "agreement": 1.0, "input_tokens": 209}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.3235069848597, "probs": [0.5631497490596038, 0.19793031332732086, 0.2389199376130753], "top": "prohibited", "agreement": 1.0, "input_tokens": 220}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.22011916711926, "probs": [0.07442810557243966, 0.7426469601625133, 0.18292493426504697], "top": "permitted", "agreement": 1.0, "input_tokens": 241}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.87431693077087, "probs": [0.021424613092898774, 0.963560742046841, 0.01501464486026032], "top": "A", "agreement": 1.0, "input_tokens": 200}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.57446295954287, "probs": [0.01601169462902707, 0.9650607227828113, 0.01892758258816167], "top": "A", "agreement": 1.0, "input_tokens": 211}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.99670582450926, "probs": [0.01772358905716697, 0.9621833982502599, 0.020093012692573263], "top": "A", "agreement": 1.0, "input_tokens": 232}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.5688740927726, "probs": [0.038484166247206124, 0.9437641516667067, 0.017751682086087198], "top": "B", "agreement": 1.0, "input_tokens": 208}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.420429026708, "probs": [0.01638885815779191, 0.9473306235405403, 0.036280518301667765], "top": "B", "agreement": 1.0, "input_tokens": 219}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14907585456967, "probs": [0.01830968230910091, 0.9532844759985375, 0.028405841692361555], "top": "B", "agreement": 1.0, "input_tokens": 240}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16815586015582, "probs": [0.023557953254122476, 0.9543166602232142, 0.022125386522663444], "top": "A", "agreement": 1.0, "input_tokens": 203}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.96405279263854, "probs": [0.031413960782038314, 0.9495687268473656, 0.01901731237059622], "top": "A", "agreement": 1.0, "input_tokens": 214}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.27796717360616, "probs": [0.07602225290100428, 0.8966935357050799, 0.027284211393915826], "top": "A", "agreement": 1.0, "input_tokens": 235}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.73298204690218, "probs": [0.023803230827026532, 0.0318997393139744, 0.944297029858999], "top": "B", "agreement": 1.0, "input_tokens": 203}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98225293122232, "probs": [0.9537738809196197, 0.029021027054749942, 0.017205092025630454], "top": "B", "agreement": 1.0, "input_tokens": 214}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22996610961854, "probs": [0.964600540916227, 0.019726610712202018, 0.015672848371570857], "top": "B", "agreement": 1.0, "input_tokens": 235}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1174949388951, "probs": [0.01331158113941839, 0.9684719956116256, 0.018216423248955952], "top": "A", "agreement": 1.0, "input_tokens": 208}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.4553990252316, "probs": [0.03555637420973125, 0.9480424533538477, 0.016401172436420974], "top": "A", "agreement": 1.0, "input_tokens": 219}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.30183697119355, "probs": [0.02192392382604651, 0.9656109109209706, 0.012465165252982892], "top": "A", "agreement": 1.0, "input_tokens": 240}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.32300709746778, "probs": [0.019595497666027976, 0.022215172638678914, 0.9581893296952932], "top": "B", "agreement": 1.0, "input_tokens": 206}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24225700646639, "probs": [0.9596984415322811, 0.02225016024721796, 0.01805139822050088], "top": "B", "agreement": 1.0, "input_tokens": 217}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.0189440138638, "probs": [0.9679216672147071, 0.013872260676720507, 0.01820607210857243], "top": "B", "agreement": 1.0, "input_tokens": 238}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.14141502790153, "probs": [0.02350526733620643, 0.04401818593722302, 0.9324765467265705], "top": "contradicted", "agreement": 1.0, "input_tokens": 216}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.22078583016992, "probs": [0.9307059696632914, 0.042134929049560656, 0.027159101287147974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16412579454482, "probs": [0.9518944397002193, 0.030200951503065683, 0.01790460879671502], "top": "contradicted", "agreement": 1.0, "input_tokens": 248}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.93329190649092, "probs": [0.0734946189438627, 0.8668797160257922, 0.05962566503034527], "top": "prohibited", "agreement": 1.0, "input_tokens": 228}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.13516492210329, "probs": [0.06641491931728377, 0.8517220237244112, 0.08186305695830509], "top": "prohibited", "agreement": 1.0, "input_tokens": 239}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.95432708598673, "probs": [0.026075603104276424, 0.9317425408247122, 0.04218185607101137], "top": "prohibited", "agreement": 1.0, "input_tokens": 260}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.91602511703968, "probs": [0.8272206536224163, 0.11345127657827278, 0.05932806979931101], "top": "A", "agreement": 1.0, "input_tokens": 216}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.37690808996558, "probs": [0.10398356939461242, 0.36466878809364145, 0.5313476425117462], "top": "A", "agreement": 0.3333333333333333, "input_tokens": 227}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24137713946402, "probs": [0.047442131153065406, 0.31816121638654793, 0.6343966524603867], "top": "A", "agreement": 1.0, "input_tokens": 248}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.25071597658098, "probs": [0.026110501601006305, 0.021183264898242012, 0.9527062335007518], "top": "contradicted", "agreement": 1.0, "input_tokens": 226}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.2236258983612, "probs": [0.9405169120595193, 0.024208959715331872, 0.035274128225148874], "top": "contradicted", "agreement": 1.0, "input_tokens": 237}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.8280139695853, "probs": [0.9538802773453129, 0.023059861528293313, 0.023059861126393875], "top": "contradicted", "agreement": 1.0, "input_tokens": 258}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.92808185517788, "probs": [0.11358428062013931, 0.09215012355248668, 0.7942655958273741], "top": "prohibited", "agreement": 1.0, "input_tokens": 233}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84474105760455, "probs": [0.6104725669434667, 0.2700592902850353, 0.11946814277149793], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 244}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.70086282677948, "probs": [0.8448473542913075, 0.08645996139090424, 0.06869268431778815], "top": "prohibited", "agreement": 1.0, "input_tokens": 265}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.98788394033909, "probs": [0.0265024583113502, 0.9469950831682415, 0.02650245852040832], "top": "A", "agreement": 1.0, "input_tokens": 222}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03404401242733, "probs": [0.04213492809402675, 0.9307059697792428, 0.027159102126730544], "top": "A", "agreement": 1.0, "input_tokens": 233}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.33639811910689, "probs": [0.02140380544189133, 0.962624913264518, 0.01597128129359056], "top": "A", "agreement": 1.0, "input_tokens": 254}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.84539205208421, "probs": [0.04515777084298974, 0.03741040508103143, 0.9174318240759788], "top": "contradicted", "agreement": 1.0, "input_tokens": 223}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.20929587073624, "probs": [0.9100932546372905, 0.041201749446211355, 0.048704995916498045], "top": "contradicted", "agreement": 1.0, "input_tokens": 234}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.28158699162304, "probs": [0.9366656772497187, 0.038194709842211357, 0.025139612908070024], "top": "contradicted", "agreement": 1.0, "input_tokens": 255}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16747599467635, "probs": [0.868308675840529, 0.07676005878826689, 0.0549312653712041], "top": "prohibited", "agreement": 1.0, "input_tokens": 224}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.18059507012367, "probs": [0.15789450752928103, 0.11538080349900755, 0.7267246889717114], "top": "prohibited", "agreement": 1.0, "input_tokens": 235}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.41363898664713, "probs": [0.08083885614456791, 0.06032105358305889, 0.8588400902723732], "top": "prohibited", "agreement": 1.0, "input_tokens": 256}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.6059910915792, "probs": [0.009236476474052145, 0.9791380440055677, 0.011625479520380164], "top": "A", "agreement": 1.0, "input_tokens": 227}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.8796430695802, "probs": [0.014594034131288612, 0.9765710625943955, 0.008834903274315823], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.90185411274433, "probs": [0.0070908823458139675, 0.986522232315347, 0.006386885338839195], "top": "A", "agreement": 1.0, "input_tokens": 259}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.1369068492204, "probs": [0.07423043782078662, 0.012813700792315847, 0.9129558613868974], "top": "contradicted", "agreement": 1.0, "input_tokens": 227}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.3259110134095, "probs": [0.9080827305344887, 0.025413463174314043, 0.06650380629119718], "top": "contradicted", "agreement": 1.0, "input_tokens": 238}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.56280708685517, "probs": [0.912724042631006, 0.024497031790175153, 0.06277892557881881], "top": "contradicted", "agreement": 1.0, "input_tokens": 259}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.38648287951946, "probs": [0.44029144656423663, 0.4049592579435585, 0.15474929549220479], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 227}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.19482793845236, "probs": [0.36520001887064335, 0.21202778871080402, 0.4227721924185527], "top": "permitted", "agreement": 0.3333333333333333, "input_tokens": 238}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.80860606953502, "probs": [0.0870101345530018, 0.5040542344698868, 0.4089356309771113], "top": "prohibited", "agreement": 0.6666666666666666, "input_tokens": 259}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.39454803802073, "probs": [0.4770731173522989, 0.04585376828987368, 0.4770731143578275], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 221}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.95601291768253, "probs": [0.39895641057043185, 0.0317669442524896, 0.5692766451770785], "top": "B", "agreement": 1.0, "input_tokens": 232}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.11420504190028, "probs": [0.3779951490486449, 0.03557902847506642, 0.5864258224762886], "top": "B", "agreement": 0.6666666666666666, "input_tokens": 253}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.99528306536376, "probs": [0.02269220266989126, 0.01879907857817761, 0.9585087187519311], "top": "contradicted", "agreement": 1.0, "input_tokens": 213}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.09157506749034, "probs": [0.9501164329419498, 0.021572126471092922, 0.028311440586957334], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.95081287808716, "probs": [0.9561058300755273, 0.022635315404623546, 0.021258854519849178], "top": "contradicted", "agreement": 1.0, "input_tokens": 245}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.17760599032044, "probs": [0.09448034891197682, 0.0384419614966505, 0.8670776895913728], "top": "prohibited", "agreement": 1.0, "input_tokens": 218}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.03115295432508, "probs": [0.7989049560014004, 0.057297274549302536, 0.143797769449297], "top": "prohibited", "agreement": 1.0, "input_tokens": 229}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.31923803500831, "probs": [0.9033714574378554, 0.03317980421079573, 0.06344873835134875], "top": "prohibited", "agreement": 1.0, "input_tokens": 250}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 104.89074210636318, "probs": [0.9807799827157122, 0.008509520309325268, 0.010710496974962463], "top": "A", "agreement": 1.0, "input_tokens": 206}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.26243690401316, "probs": [0.009842123709939055, 0.01026250243418525, 0.9798953738558757], "top": "A", "agreement": 1.0, "input_tokens": 217}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.82304303534329, "probs": [0.011627702527445261, 0.00904705287357599, 0.9793252445989787], "top": "A", "agreement": 1.0, "input_tokens": 238}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.35805812105536, "probs": [0.6618883595686608, 0.28081044656836285, 0.05730119386297648], "top": "contradicted", "agreement": 1.0, "input_tokens": 224}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.16671487130225, "probs": [0.06488320495606428, 0.37587178359700457, 0.5592450114469311], "top": "contradicted", "agreement": 1.0, "input_tokens": 235}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.12407496571541, "probs": [0.06100397866886853, 0.1887116244179442, 0.7502843969131873], "top": "contradicted", "agreement": 1.0, "input_tokens": 256}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.24153592996299, "probs": [0.17125015092489698, 0.5080506282282206, 0.32069922084688257], "top": "permitted", "agreement": 0.6666666666666666, "input_tokens": 226}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.60784093104303, "probs": [0.1514529519738628, 0.627869626193126, 0.22067742183301117], "top": "permitted", "agreement": 1.0, "input_tokens": 237}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.07885709032416, "probs": [0.17013095481276255, 0.7202069629772057, 0.10966208221003176], "top": "permitted", "agreement": 1.0, "input_tokens": 258}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.49077291600406, "probs": [0.023818955965634256, 0.9449208465220887, 0.031260197512277065], "top": "A", "agreement": 1.0, "input_tokens": 213}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.1828758791089, "probs": [0.04244531776287583, 0.9375620734741275, 0.01999260876299673], "top": "A", "agreement": 1.0, "input_tokens": 224}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.86997400969267, "probs": [0.028930382730149296, 0.9507948329241313, 0.02027478434571944], "top": "A", "agreement": 1.0, "input_tokens": 245}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86626503244042, "probs": [0.227340819830238, 0.7726591801697621], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18090707808733, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06262787804008, "probs": [0.16800751383797227, 0.8319924861620277], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.33539905771613, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86905620619655, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.39100879430771, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.32165902294219, "probs": [0.20097170058635486, 0.7990282994136452], "top": "ordinary", "agreement": 1.0, "input_tokens": 254}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06777785904706, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0551279373467, "probs": [0.13950622912843208, 0.8604937708715679], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4328798931092, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.86461519449949, "probs": [0.3067002008775075, 0.6932997991224925], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.229426888749, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.944085881114, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "agreement": 1.0, "input_tokens": 256}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25466805882752, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99205690994859, "probs": [0.07740461434958126, 0.9225953856504188], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11000595614314, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.61531201563776, "probs": [0.09196032674706488, 0.9080396732529351], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.08513591811061, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.82638510130346, "probs": [0.1552608265624769, 0.844739173437523], "top": "ordinary", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.23083784244955, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64636207744479, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.16289692930877, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.45620998926461, "probs": [0.32019975839988934, 0.6798002416001107], "top": "ordinary", "agreement": 1.0, "input_tokens": 255}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.18803611956537, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67256297543645, "probs": [0.10298814543711346, 0.8970118545628866], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.53207086957991, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62682199850678, "probs": [0.10012611338996007, 0.8998738866100399], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.21526706032455, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66413310915232, "probs": [0.13578298208575287, 0.8642170179142472], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.13187597133219, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.57731195911765, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22672791592777, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71707616746426, "probs": [0.9710643563300382, 0.02893564366996184], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.29292818717659, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.25169008597732, "probs": [0.9673224283590844, 0.032677571640915566], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.26700808294117, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.99369790032506, "probs": [0.9350120677910418, 0.06498793220895807], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27948803827167, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90570584498346, "probs": [0.8405803537312423, 0.1594196462687578], "top": "react", "agreement": 1.0, "input_tokens": 264}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.54220109805465, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.1125090494752, "probs": [0.9290542127600319, 0.07094578723996812], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.06852601282299, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.513542836532, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.00813393481076, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72066299244761, "probs": [0.9516645319845058, 0.048335468015494064], "top": "react", "agreement": 1.0, "input_tokens": 259}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27767917700112, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03118689358234, "probs": [0.9439168350492196, 0.056083164950780265], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72017288394272, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43481099046767, "probs": [0.905386513027929, 0.09461348697207088], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.94557599723339, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.51935405656695, "probs": [0.9156019647894573, 0.0843980352105427], "top": "react", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4836809784174, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31442094780505, "probs": [0.6155100736384166, 0.38448992636158347], "top": "react", "agreement": 1.0, "input_tokens": 262}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.24630983546376, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.39514193497598, "probs": [0.6932998086882517, 0.30670019131174836], "top": "react", "agreement": 1.0, "input_tokens": 268}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.96631701663136, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.20541010610759, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 260}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.91460602916777, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.93324917554855, "probs": [0.915601962671152, 0.08439803732884785], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.5416120365262, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43232289329171, "probs": [0.40698895046193323, 0.5930110495380667], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06524815410376, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.95294601842761, "probs": [0.36247441705939704, 0.637525582940603], "top": "ordinary", "agreement": 1.0, "input_tokens": 257}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.85330521315336, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.09998811595142, "probs": [0.908039679856927, 0.09196032014307312], "top": "react", "agreement": 1.0, "input_tokens": 261}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.87348511070013, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.45882297866046, "probs": [0.3481052260909569, 0.6518947739090432], "top": "ordinary", "agreement": 1.0, "input_tokens": 258}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28280091658235, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90084620378911, "probs": [0.5235093404303358, 0.4764906595696642], "top": "react", "agreement": 0.5, "input_tokens": 255}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8387661036104, "probs": [0.24429382031551497, 0.7557061796844852], "top": "ordinary", "agreement": 1.0, "input_tokens": 250}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 76.22286002151668, "probs": [0.2743951410804348, 0.7256048589195652], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.47663289308548, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.42274011857808, "probs": [0.2112354509623397, 0.7887645490376604], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.75354403443635, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.21674011088908, "probs": [0.12511350715617336, 0.8748864928438266], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.69518410228193, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.81886490434408, "probs": [0.27439514286499256, 0.7256048571350073], "top": "ordinary", "agreement": 1.0, "input_tokens": 181}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63280202075839, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9403859693557, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76996488124132, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66156289167702, "probs": [0.4222145135481339, 0.5777854864518661], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77340402267873, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.55294204130769, "probs": [0.1472092305510008, 0.8527907694489993], "top": "ordinary", "agreement": 1.0, "input_tokens": 183}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63825305178761, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84939505532384, "probs": [0.11201237833636817, 0.8879876216636319], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28051009587944, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.45914086140692, "probs": [0.12172004831180805, 0.878279951688192], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.82387316040695, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.44548201560974, "probs": [0.17243807104776526, 0.8275619289522348], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30926800891757, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.81735406629741, "probs": [0.16366828025224447, 0.8363317197477556], "top": "ordinary", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19656021334231, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.35790090635419, "probs": [0.3000708042310285, 0.6999291957689716], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76091405563056, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89006498083472, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.78742485120893, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.237340034917, "probs": [0.10592235643160026, 0.8940776435683997], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.62408297881484, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79205399006605, "probs": [0.19108583142179145, 0.8089141685782085], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16640911065042, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.4707300066948, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38593913987279, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.68244291096926, "probs": [0.7614507841997993, 0.23854921580020058], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.03702791593969, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.87584601342678, "probs": [0.7887645476620578, 0.2112354523379421], "top": "react", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.79003511555493, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.16139906086028, "probs": [0.4921584130878123, 0.5078415869121877], "top": "ordinary", "agreement": 0.5, "input_tokens": 191}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56618218868971, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.10097904317081, "probs": [0.293524032474578, 0.7064759675254221], "top": "ordinary", "agreement": 1.0, "input_tokens": 191}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80322404019535, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77868392318487, "probs": [0.5156793119007109, 0.484320688099289], "top": "react", "agreement": 0.5, "input_tokens": 186}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48462091572583, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.64348312653601, "probs": [0.6302458313412554, 0.3697541686587445], "top": "react", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.19901983626187, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73358416184783, "probs": [0.7318060925310856, 0.2681939074689144], "top": "react", "agreement": 1.0, "input_tokens": 186}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.01806509308517, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0666078850627, "probs": [0.5469149637193962, 0.45308503628060387], "top": "react", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.54256308078766, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.9168761279434, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.69583300128579, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90122501924634, "probs": [0.44532390802961846, 0.5546760919703815], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.01825890317559, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89125590957701, "probs": [0.22187782586721538, 0.7781221741327846], "top": "ordinary", "agreement": 1.0, "input_tokens": 189}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.74331392161548, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.06133799627423, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "agreement": 1.0, "input_tokens": 195}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.67149288579822, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.77398400381207, "probs": [0.2501313189948441, 0.749868681005156], "top": "ordinary", "agreement": 1.0, "input_tokens": 187}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.55322097241879, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04088801518083, "probs": [0.17696061199867663, 0.8230393880013234], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.27876789309084, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84461504220963, "probs": [0.2620821975193846, 0.7379178024806153], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.8658900372684, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97037891112268, "probs": [0.2870615213045163, 0.7129384786954838], "top": "ordinary", "agreement": 1.0, "input_tokens": 184}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.64604691974819, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.2792119961232, "probs": [0.4145813330827104, 0.5854186669172896], "top": "ordinary", "agreement": 1.0, "input_tokens": 188}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.53519508615136, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.50798603147268, "probs": [0.44532391331269805, 0.554676086687302], "top": "ordinary", "agreement": 1.0, "input_tokens": 185}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.92307091504335, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.713748017326, "probs": [0.35525721497515955, 0.6447427850248404], "top": "ordinary", "agreement": 1.0, "input_tokens": 182}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.62807588651776, "probs": [0.17696061753450526, 0.8230393824654948], "top": "ordinary", "agreement": 1.0, "input_tokens": 177}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.09194500744343, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.95268115215003, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.23944097198546, "probs": [0.8269315768853935, 0.12075476980815658, 0.05231365330644987], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.83213090710342, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.75483091548085, "probs": [0.9608203282304785, 0.03917967176952151], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.76461918465793, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.58605408854783, "probs": [0.9620325019409397, 0.02528607877990563, 0.012681419279154706], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.28979702293873, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.28705708682537, "probs": [0.06890580795431694, 0.931094192045683], "top": "holds", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.4181550797075, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.99289800040424, "probs": [0.8320645593484999, 0.04643111322039759, 0.12150432743110248], "top": "stay", "agreement": 1.0, "input_tokens": 402}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.01373415254056, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.01612417399883, "probs": [0.9544700034284429, 0.04552999657155723], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.08799300901592, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 110.85405596531928, "probs": [0.9527150999124248, 0.019483546419057494, 0.027801353668517746], "top": "stay", "agreement": 1.0, "input_tokens": 414}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.2643119841814, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.85603787750006, "probs": [0.12172004535178665, 0.8782799546482134], "top": "holds", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.32225104235113, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.85101999156177, "probs": [0.0645643144235774, 0.035205281163747985, 0.9002304044126745], "top": "tavern", "agreement": 1.0, "input_tokens": 400}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.19203609228134, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 75.08493517525494, "probs": [0.1862839992992625, 0.8137160007007376], "top": "elsewhere", "agreement": 1.0, "input_tokens": 360}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.7175230383873, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.42717601917684, "probs": [0.36775256751831553, 0.024771247006193476, 0.6074761854754909], "top": "tavern", "agreement": 0.6666666666666666, "input_tokens": 412}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.485391035676, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.44636507332325, "probs": [0.0796746792375593, 0.9203253207624406], "top": "holds", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.56509184278548, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.88804099522531, "probs": [0.08290919036094808, 0.8994500391535621, 0.017640770485489753], "top": "meet_wren", "agreement": 1.0, "input_tokens": 403}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.08378704637289, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.001200100407, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 363}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.36108912527561, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.47540711984038, "probs": [0.6296525959845715, 0.35799812782967866, 0.012349276185749847], "top": "stay", "agreement": 0.6666666666666666, "input_tokens": 415}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.49111799709499, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.94767908751965, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 351}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.63391192443669, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.78367705643177, "probs": [0.5514055185500297, 0.41145265329172886, 0.03714182815824141], "top": "stay", "agreement": 1.0, "input_tokens": 404}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.52265006117523, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.83850712701678, "probs": [0.9673224283553485, 0.03267757164465143], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.52882101945579, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.98393003642559, "probs": [0.9603331749383929, 0.028023658411333054, 0.011643166650274083], "top": "stay", "agreement": 1.0, "input_tokens": 416}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.1606181319803, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.748415870592, "probs": [0.7439389594906934, 0.2560610405093065], "top": "moved", "agreement": 1.0, "input_tokens": 345}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47655100561678, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.73777798749506, "probs": [0.329927824572513, 0.5565122407492623, 0.11355993467822469], "top": "meet_wren", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.16289800964296, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87464896216989, "probs": [0.02983020500731863, 0.9701697949926814], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.29106787033379, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.08253102004528, "probs": [0.7871268373024943, 0.18209101999191218, 0.030782142705593513], "top": "stay", "agreement": 1.0, "input_tokens": 410}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 106.2692292034626, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.7843469940126, "probs": [0.04979902878533396, 0.950200971214666], "top": "holds", "agreement": 1.0, "input_tokens": 334}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.38704881072044, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 107.42668411694467, "probs": [0.9109478037576992, 0.057628718316645906, 0.03142347792565483], "top": "stay", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 105.42142880149186, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.79346899688244, "probs": [0.9719328658769364, 0.028067134123063546], "top": "same", "agreement": 1.0, "input_tokens": 347}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.41697509400547, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.36130196973681, "probs": [0.9710720272034978, 0.01715460731299303, 0.01177336548350914], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.11364504136145, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.6876010671258, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.31203296780586, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.73779610544443, "probs": [0.1337836073050901, 0.8426345520770127, 0.0235818406178973], "top": "meet_wren", "agreement": 1.0, "input_tokens": 407}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 108.26032399199903, "probs": [0.8543437299733625, 0.08041563554402914, 0.06524063448260843], "top": "stay", "agreement": 1.0, "input_tokens": 256}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.3173558730632, "probs": [0.9516645330159608, 0.048335466984039115], "top": "same", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.5187151003629, "probs": [0.8275619297045159, 0.1724380702954841], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 111.52179399505258, "probs": [0.7852826544019835, 0.20168841824349107, 0.013028927354525465], "top": "stay", "agreement": 1.0, "input_tokens": 419}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 109.05481385998428, "probs": [0.9448144437724326, 0.03328047439527607, 0.02190508183229131], "top": "stay", "agreement": 1.0, "input_tokens": 268}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.50320711359382, "probs": [0.05776707671018535, 0.9422329232898147], "top": "holds", "agreement": 1.0, "input_tokens": 336}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.63797700963914, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 74.76186100393534, "probs": [0.706475960089318, 0.29352403991068193], "top": "stay", "agreement": 1.0, "input_tokens": 370}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.90194609574974, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.56751297973096, "probs": [0.9631149772558724, 0.036885022744127596], "top": "same", "agreement": 1.0, "input_tokens": 349}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.96058705449104, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.21833190508187, "probs": [0.9583890947004433, 0.04161090529955677], "top": "stay", "agreement": 1.0, "input_tokens": 381}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.86197403259575, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.73779599927366, "probs": [0.05608316364845092, 0.9439168363515491], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.55039091594517, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87953817285597, "probs": [0.5156793134278208, 0.4843206865721793], "top": "stay", "agreement": 0.5, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.23427814804018, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.74265703745186, "probs": [0.9471470205154783, 0.05285297948452184], "top": "same", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.10826601274312, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.95091915875673, "probs": [0.9471470206637833, 0.05285297933621657], "top": "stay", "agreement": 1.0, "input_tokens": 389}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.15669688209891, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.28044094517827, "probs": [0.06310762735784174, 0.9368923726421583], "top": "holds", "agreement": 1.0, "input_tokens": 340}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.59065292216837, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.46572291478515, "probs": [0.7614507825686178, 0.23854921743138224], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43584988079965, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.88489816710353, "probs": [0.9583890946533792, 0.04161090534662085], "top": "same", "agreement": 1.0, "input_tokens": 353}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.97779696434736, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.18967208266258, "probs": [0.9516645320340041, 0.04833546796599583], "top": "stay", "agreement": 1.0, "input_tokens": 385}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.48448098450899, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.21024995669723, "probs": [0.4921584129984156, 0.5078415870015844], "top": "holds", "agreement": 1.0, "input_tokens": 333}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25595794059336, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.65167497098446, "probs": [0.07094578575062087, 0.9290542142493791], "top": "departure", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.12859608605504, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.42042291909456, "probs": [0.0820053924809899, 0.9179946075190101], "top": "elsewhere", "agreement": 1.0, "input_tokens": 346}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.72177382744849, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.8624181356281, "probs": [0.1724380702954841, 0.8275619297045159], "top": "departure", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.66487304493785, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.83396692946553, "probs": [0.26819391433993306, 0.7318060856600669], "top": "holds", "agreement": 1.0, "input_tokens": 339}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.80802500806749, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.86173803731799, "probs": [0.04833546845435023, 0.9516645315456497], "top": "departure", "agreement": 1.0, "input_tokens": 373}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.22348807752132, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.57432397454977, "probs": [0.06310762735784174, 0.9368923726421583], "top": "elsewhere", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.14892592653632, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.53668317571282, "probs": [0.17243806760921704, 0.8275619323907829], "top": "departure", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.24607707373798, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.83271802589297, "probs": [0.06692025896742367, 0.9330797410325763], "top": "holds", "agreement": 1.0, "input_tokens": 337}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.84839504770935, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.87052809260786, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "agreement": 1.0, "input_tokens": 371}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.49153994396329, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 73.02488898858428, "probs": [0.9608203281857868, 0.039179671814213245], "top": "same", "agreement": 1.0, "input_tokens": 350}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11817598342896, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98541907221079, "probs": [0.9439168372028508, 0.05608316279714923], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.89450599253178, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36770214512944, "probs": [0.07740461992444837, 0.9225953800755516], "top": "holds", "agreement": 1.0, "input_tokens": 343}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.0111271366477, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63476517982781, "probs": [0.6589789156058212, 0.3410210843941787], "top": "stay", "agreement": 1.0, "input_tokens": 377}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.8593928720802, "probs": [0.85668535789068, 0.14331464210931996], "top": "stay", "agreement": 1.0, "input_tokens": 237}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43625191040337, "probs": [0.9596222367905244, 0.040377763209475684], "top": "same", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.68732397258282, "probs": [0.8137160007007376, 0.1862839992992625], "top": "same", "agreement": 1.0, "input_tokens": 216}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.98649894073606, "probs": [0.9422329206296081, 0.05776707937039187], "top": "stay", "agreement": 1.0, "input_tokens": 388}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.1833368986845, "probs": [0.9310941957223424, 0.06890580427765759], "top": "stay", "agreement": 1.0, "input_tokens": 248}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.73719599470496, "probs": [0.061278160236647276, 0.9387218397633527], "top": "holds", "agreement": 1.0, "input_tokens": 352}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.73247402533889, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.7128149010241, "probs": [0.8910699865839312, 0.10893001341606875], "top": "stay", "agreement": 1.0, "input_tokens": 382}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.090656099841, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76285695843399, "probs": [0.9663161960398492, 0.03368380396015082], "top": "same", "agreement": 1.0, "input_tokens": 362}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.8460560310632, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.9207880795002, "probs": [0.9673224283958624, 0.03267757160413767], "top": "stay", "agreement": 1.0, "input_tokens": 394}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.43080001696944, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.81554699875414, "probs": [0.8527907664515249, 0.14720923354847515], "top": "moved", "agreement": 1.0, "input_tokens": 348}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.76110404543579, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 77.26746192201972, "probs": [0.04161090472473705, 0.9583890952752631], "top": "outside", "agreement": 1.0, "input_tokens": 378}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.04029709100723, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76070700027049, "probs": [0.05608316279714923, 0.9439168372028508], "top": "elsewhere", "agreement": 1.0, "input_tokens": 358}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.96910383552313, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.99359887838364, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "agreement": 1.0, "input_tokens": 390}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.419128915295, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.97074887901545, "probs": [0.07094578723996812, 0.9290542127600319], "top": "holds", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.15635694935918, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63877498917282, "probs": [0.8040183777913431, 0.19598162220865686], "top": "stay", "agreement": 1.0, "input_tokens": 384}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.100695990026, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.31223117560148, "probs": [0.959622237525547, 0.040377762474453], "top": "same", "agreement": 1.0, "input_tokens": 364}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.25087804161012, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.79137684963644, "probs": [0.9583890941641539, 0.041610905835846104], "top": "stay", "agreement": 1.0, "input_tokens": 396}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.11810683272779, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.33206112869084, "probs": [0.7781221683337017, 0.22187783166629826], "top": "moved", "agreement": 1.0, "input_tokens": 357}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.30091800354421, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.63838406652212, "probs": [0.21123545145303588, 0.7887645485469642], "top": "outside", "agreement": 1.0, "input_tokens": 387}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.0087651386857, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.43410195223987, "probs": [0.03688502246073314, 0.9631149775392668], "top": "elsewhere", "agreement": 1.0, "input_tokens": 367}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.09778607264161, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.8235871065408, "probs": [0.755706186939545, 0.244293813060455], "top": "stay", "agreement": 1.0, "input_tokens": 399}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.92706300318241, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.23281892947853, "probs": [0.12172004614068278, 0.8782799538593171], "top": "holds", "agreement": 1.0, "input_tokens": 356}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.47922017611563, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.35892093740404, "probs": [0.8879876230204682, 0.11201237697953184], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.08879601582885, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.69162498414516, "probs": [0.9558139330893122, 0.04418606691068789], "top": "same", "agreement": 1.0, "input_tokens": 366}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.8522628992796, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.71660608239472, "probs": [0.9631149782667298, 0.0368850217332701], "top": "stay", "agreement": 1.0, "input_tokens": 398}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.094956016168, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.36917107366025, "probs": [0.09733498488397495, 0.9026650151160249], "top": "holds", "agreement": 1.0, "input_tokens": 344}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.10607600770891, "probs": [0.11517068726201778, 0.8848293127379823], "top": "holds", "agreement": 1.0, "input_tokens": 203}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.62737490236759, "probs": [0.8319924818822048, 0.16800751811779524], "top": "stay", "agreement": 1.0, "input_tokens": 374}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.13582594320178, "probs": [0.8879876250021805, 0.1120123749978195], "top": "stay", "agreement": 1.0, "input_tokens": 233}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.53120304085314, "probs": [0.9471470206637833, 0.05285297933621657], "top": "same", "agreement": 1.0, "input_tokens": 354}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.52480895072222, "probs": [0.8527907646474562, 0.14720923535254382], "top": "same", "agreement": 1.0, "input_tokens": 213}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 72.76193611323833, "probs": [0.955813931654481, 0.044186068345518914], "top": "stay", "agreement": 1.0, "input_tokens": 386}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 71.05968496762216, "probs": [0.908039676626573, 0.09196032337342702], "top": "stay", "agreement": 1.0, "input_tokens": 245}], "seconds": 50.1, "failures": 0}, "repeat": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.23254916258156, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 34.15}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.028327118605375, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 34.03}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03186591155827, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 34.03}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.162038031965494, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 34.23}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.879763938486576, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 33.94}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05266700126231, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 34.12}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86464414931834, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 33.88}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.47336193919182, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 34.55}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16909815557301, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 34.09}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.13988699018955, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 34.19}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09222702309489, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.16}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16548811644316, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 34.2}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.02711500227451, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 34.08}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1583579108119, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 34.2}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80704301223159, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 33.89}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.313819862902164, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 34.37}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94368596188724, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 34.04}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.175297969952226, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 34.19}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95994499139488, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 34.05}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.291509099304676, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 34.37}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94928507320583, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 33.95}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89924511127174, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 33.99}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.94680496118963, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 34.01}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.51171287149191, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 34.56}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.78108309395611, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 33.8}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.217289209365845, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 34.29}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.37996006384492, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 34.4}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.123268004506826, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 34.22}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.67255211621523, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 33.79}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.406891118735075, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 34.4}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.506030010059476, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 33.6}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.16443805024028, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 34.23}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59661998972297, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 33.66}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05005603842437, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 34.0}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01518615707755, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.09}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.56603295914829, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 34.55}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08721697330475, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 34.1}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.09544683806598, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 34.21}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.922294879332185, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 33.91}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05770708620548, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 34.13}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.716622998937964, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 33.72}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0760359801352, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 34.18}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.112477933987975, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 34.17}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.50309208407998, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 34.56}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.76608297973871, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 33.83}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.208828842267394, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 34.24}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.56734013929963, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 33.64}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.06947714090347, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 34.1}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.65739407390356, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 34.73}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.596262991428375, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 34.6}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.999445874243975, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.07}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.38411096855998, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 34.46}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.116636987775564, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 34.16}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1012669056654, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 34.2}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.93158505484462, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 34.02}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20871894620359, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 34.24}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.114576902240515, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 34.13}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.106527015566826, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 34.22}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.8676339276135, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.05}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.2333291452378, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 34.3}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.712421802803874, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 33.89}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.49992209486663, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 34.7}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.65614197961986, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 33.74}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0118860155344, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 34.23}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.57118999399245, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 33.9}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.19130903482437, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 34.45}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.290109088644385, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 34.4}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.366561125963926, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 34.43}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31977893039584, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 34.45}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.452901946380734, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 34.57}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80878388695419, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 34.02}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.17176802456379, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 34.27}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.97552508674562, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 34.14}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.03739587217569, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 34.19}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.830243887379766, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 33.89}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20806911587715, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 34.55}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.533589009195566, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 34.03}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.25649998523295, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 34.75}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.75839211791754, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 34.22}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05988684669137, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 34.58}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.710422951728106, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 34.21}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05339599400759, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 34.55}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.53504000976682, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 34.05}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.12036716751754, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 34.54}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.691071931272745, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 34.19}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.20709797926247, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 34.69}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.83449397794902, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 34.35}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.119027925655246, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 34.56}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.00619516894221, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 34.49}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.26585884578526, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 34.6}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.0895869564265, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.41}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.283570040017366, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 34.68}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.995964823290706, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 34.43}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.386670941486955, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 34.66}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.835734033957124, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 33.92}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15450712479651, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 34.32}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87222511507571, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 34.01}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.21134806983173, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 34.53}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.723172990605235, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 34.21}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.71010506898165, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 35.2}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.89850401133299, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 34.23}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.31600008904934, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 34.63}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.025655852630734, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 34.44}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.59711282327771, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 34.9}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.006615893915296, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 34.39}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.18516789376736, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 34.5}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01397590339184, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 34.29}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.299609975889325, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 34.65}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.05335594713688, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 34.42}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.1631879825145, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 34.51}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.970836110413074, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.28}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.24760003387928, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 34.64}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.01944509521127, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 34.31}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.265409948304296, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 34.59}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84754390269518, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 34.16}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.43252111598849, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 34.8}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.91513487137854, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.22}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.264188984408975, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 34.64}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.80208302102983, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.25}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.41681110113859, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 34.76}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.902745021507144, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 34.32}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3522499576211, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 34.67}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.07059612497687, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 34.37}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.420832086354494, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 34.76}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.055006017908454, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 34.39}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.182188127189875, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 34.51}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.08865703083575, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 34.39}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.3034100048244, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 34.63}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.79778310470283, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 34.12}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.15852787718177, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 34.49}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.95087507367134, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 34.32}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.33392991125584, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 34.74}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.754903150722384, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 34.14}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.285798927769065, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 34.68}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.74098290503025, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 34.24}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.96481604129076, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 34.49}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11859695613384, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 34.23}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.44849096983671, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 34.77}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.59725096076727, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 34.06}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 37.11113706231117, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 34.61}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.86432493850589, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 34.36}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.99478507041931, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 34.49}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.87612502835691, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.39}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 36.84546402655542, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 34.32}], "seconds": 5.4, "failures": 0}, "negative": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.882923915982246, "probs": [0.06566437538338366, 0.8598927385077062, 0.07444288610891006], "top": "insufficient", "option_logits": null, "prompt_sha256": "2111f6636d10660f740e0f032908924d0640e60a712060af19c6e8972d95e5c9", "input_tokens": 204, "server_ms": 34.37}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.292150082066655, "probs": [0.2945662267563794, 0.03956295740237862, 0.665870815841242], "top": "contradicted", "option_logits": null, "prompt_sha256": "ed5aa8d62ce8778e40b8c51f1d1fe90b1f392ab252247908adc4e84b2546ab08", "input_tokens": 202, "server_ms": 34.76}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.66695184074342, "probs": [0.9241406947470572, 0.051569175242271295, 0.024290130010671554], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6f696700120ee981b3e3e0dc237f67896db84867803273e124be96bfd72fecc", "input_tokens": 196, "server_ms": 34.16}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.28761896491051, "probs": [0.11215130215053613, 0.052825544241278395, 0.8350231536081855], "top": "contradicted", "option_logits": null, "prompt_sha256": "ef4b71a8ae0a1bc3711450f77606ddabd2be62c9c2b6587e1fcacbccfcb9cb00", "input_tokens": 196, "server_ms": 34.73}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.765523022040725, "probs": [0.15307015041158983, 0.014109646797217717, 0.8328202027911924], "top": "insufficient", "option_logits": null, "prompt_sha256": "215840810bc2483aa303cd79f5d4f92093a49aa9957dfa697f20989aa1390d55", "input_tokens": 209, "server_ms": 34.25}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.00892603956163, "probs": [0.03329916574926079, 0.8166100641505288, 0.1500907701002104], "top": "supported", "option_logits": null, "prompt_sha256": "fc0d6607f95e1b53fe2cfa889a2cf1bfed2a95e2da0424c85742050fd8b5cde7", "input_tokens": 206, "server_ms": 34.5}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.63306194357574, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "supported", "option_logits": null, "prompt_sha256": "f3bf4ddaa8f4178e0b5f59c09fffe0c07bfd4bd34d616bbf9577361257ca1afa", "input_tokens": 208, "server_ms": 34.07}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.06369595602155, "probs": [0.3541639151292647, 0.5160413123689802, 0.1297947725017551], "top": "contradicted", "option_logits": null, "prompt_sha256": "e5578ff7fa77b4e9f8593ede9b8e31ecccdca126f02444f3c94cdd0bb01ee6ca", "input_tokens": 200, "server_ms": 34.51}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.70328203588724, "probs": [0.017835936327419456, 0.8721495360638406, 0.11001452760873995], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b2ae5222808313b49d4b84a395fe3c5b48345949fa231e2c8fba100ed470a94", "input_tokens": 194, "server_ms": 34.14}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.26375917904079, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b42ed76cec9fa4a467694c0069d312883b296f2ee2385059235881dee283be", "input_tokens": 194, "server_ms": 34.72}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.780972965061665, "probs": [0.03186734083364, 0.0247947114952375, 0.9433379476711226], "top": "contradicted", "option_logits": null, "prompt_sha256": "d2c60f4c3e6615ad87fdb4cd474622ea0a7f3f3b3fd7df582b5539d02a9dbd1b", "input_tokens": 193, "server_ms": 34.3}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.11374709382653, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "contradicted", "option_logits": null, "prompt_sha256": "89f547d1f4dbfccb3c2e3443f18b4576344bd16efeaec8108d2febb0c913ec9e", "input_tokens": 192, "server_ms": 34.55}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.77100292406976, "probs": [0.03540454160362492, 0.924457767599498, 0.04013769079687701], "top": "contradicted", "option_logits": null, "prompt_sha256": "6beeb2f79fa6330dc68f2b769cf2779997d4b57c459bf40113bbc11a80353072", "input_tokens": 205, "server_ms": 34.28}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.98971588164568, "probs": [0.028539740573724572, 0.7452090392508762, 0.22625122017539928], "top": "insufficient", "option_logits": null, "prompt_sha256": "b094a32aab8f6f92e09b2405572effe6bec478d6d3b5985cdeb71dab794ed591", "input_tokens": 205, "server_ms": 34.41}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.95827513001859, "probs": [0.026796810283021767, 0.844600378585805, 0.12860281113117333], "top": "insufficient", "option_logits": null, "prompt_sha256": "4e199755434441474e22e920ad5f8587827e30212d91945271ceadbb26337008", "input_tokens": 199, "server_ms": 34.43}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.26776922121644, "probs": [0.1303439116251751, 0.013621023271142042, 0.8560350651036829], "top": "contradicted", "option_logits": null, "prompt_sha256": "d75aa0e009db26e5e38e3bfd553573eccf95a0526360da23bada050f23e908f8", "input_tokens": 194, "server_ms": 34.7}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.7253830190748, "probs": [0.1369403800499024, 0.018392355065480374, 0.8446672648846173], "top": "supported", "option_logits": null, "prompt_sha256": "9e994209df771b9f8829dd54c36907e41ed241a507cce230c7e3505302a9fa1e", "input_tokens": 198, "server_ms": 34.26}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.44961088523269, "probs": [0.7651520472632017, 0.10276696393112311, 0.13208098880567515], "top": "contradicted", "option_logits": null, "prompt_sha256": "45785396c292fe5cbe62d99a45714d49c993d6f5b5783769cec1795f7b84f35f", "input_tokens": 198, "server_ms": 34.8}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.95624601095915, "probs": [0.9249067812606918, 0.05495367443647756, 0.020139544302830666], "top": "contradicted", "option_logits": null, "prompt_sha256": "490eb806b0921b46aa816c18534ea40c0719e89eb76c34feb4e89d55c921bf36", "input_tokens": 194, "server_ms": 34.4}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.182107800617814, "probs": [0.0190993007707109, 0.7736996936098133, 0.2072010056194758], "top": "contradicted", "option_logits": null, "prompt_sha256": "7e29e327fc2aef321d690607a7c7347871c7eb3dd07ac6d45327aab094603d91", "input_tokens": 194, "server_ms": 34.64}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.70512209646404, "probs": [0.021206655711830842, 0.5200601388697108, 0.45873320541845836], "top": "insufficient", "option_logits": null, "prompt_sha256": "ffb27e8dfc1d8298a601b7f483c42104d68f6e8ebfea086896afd140756b8c6c", "input_tokens": 211, "server_ms": 34.22}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.09609713405371, "probs": [0.9097361205627902, 0.05076536678270011, 0.039498512654509606], "top": "supported", "option_logits": null, "prompt_sha256": "e7855c00f037441fcdc5ff3fefaa61ddaeb0e79e09ba6c0c6043c767ee6d98ca", "input_tokens": 211, "server_ms": 34.53}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.713691893965006, "probs": [0.9223434223370909, 0.04004589025644397, 0.03761068740646525], "top": "supported", "option_logits": null, "prompt_sha256": "1a0c7ecb563009263a7969e56c6e00fc34f99b8ad08b4db579ed9e2bc78f4c6f", "input_tokens": 201, "server_ms": 34.23}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.22501895390451, "probs": [0.1051237595843464, 0.007539451491827663, 0.8873367889238258], "top": "contradicted", "option_logits": null, "prompt_sha256": "610dae3d86ea66ae02b343c18c1478ef66be2619fe3a6c03ff05a434fac307d6", "input_tokens": 197, "server_ms": 34.74}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.568820010870695, "probs": [0.7409240241997488, 0.18635729477403776, 0.07271868102621346], "top": "insufficient", "option_logits": null, "prompt_sha256": "52c15cca48120ff357156a28827735982f0407b36b9806dc79a94193e52ed2a4", "input_tokens": 212, "server_ms": 34.1}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.00299700722098, "probs": [0.9070425995126343, 0.07852475778895956, 0.01443264269840612], "top": "prohibited", "option_logits": null, "prompt_sha256": "5df6c8de75514db23d6e53210ba8920e11a83a08e31ffd76867eb6d789db3f29", "input_tokens": 218, "server_ms": 34.52}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.838924046605825, "probs": [0.04968326826324694, 0.890344493113758, 0.05997223862299509], "top": "insufficient", "option_logits": null, "prompt_sha256": "3dc91e8f64358377c40956ef40870819732d6e66fc8ed44530ee8b7b1e32f1e6", "input_tokens": 203, "server_ms": 34.29}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.92587395198643, "probs": [0.09374299764974467, 0.009200493913293934, 0.8970565084369614], "top": "prohibited", "option_logits": null, "prompt_sha256": "866701419f4e54bef33f2151b0e4c0c830728dbfd94c6ba4a3fbd92b438151a2", "input_tokens": 209, "server_ms": 34.38}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.738652968779206, "probs": [0.20575303419639995, 0.025954061735184695, 0.7682929040684153], "top": "prohibited", "option_logits": null, "prompt_sha256": "6c14eaaad2ab386d458a6458fce2b79bcd81ecbfc3c66679c950d808dd0c2a3f", "input_tokens": 209, "server_ms": 34.28}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.79768298752606, "probs": [0.8315971903507323, 0.10489918128594154, 0.06350362836332618], "top": "permitted", "option_logits": null, "prompt_sha256": "d8b705bc82e40643b8dcd135b43f67a8e3671ecc2f28e81b48de6aa1cbb84792", "input_tokens": 214, "server_ms": 34.3}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.95413609966636, "probs": [0.258643657502053, 0.6628301324407543, 0.07852621005719278], "top": "insufficient", "option_logits": null, "prompt_sha256": "5000e3b05641e28823047872c8bd19f4604d76e0fa38c23bdad3d8e7be7bce61", "input_tokens": 209, "server_ms": 34.49}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.17563790269196, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "prohibited", "option_logits": null, "prompt_sha256": "97ddaad2b5074dab6f31a5a7a8ded12c3e51123aa5d819f19c006ba123fde089", "input_tokens": 209, "server_ms": 34.67}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.768823163583875, "probs": [0.09658208517581993, 0.868022296339126, 0.03539561848505407], "top": "insufficient", "option_logits": null, "prompt_sha256": "5783bf781c97c761780f8d8454e12c8895251116d4d5ebe264401d9d9dcae0a1", "input_tokens": 208, "server_ms": 34.29}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.11969708092511, "probs": [0.0651099549445377, 0.02705164492548852, 0.9078384001299739], "top": "prohibited", "option_logits": null, "prompt_sha256": "a92081d320caa09114cab155b336249985672a93e86328503ed4397aa61c3293", "input_tokens": 211, "server_ms": 34.46}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.08112589083612, "probs": [0.049132112383896635, 0.729412594713007, 0.22145529290309635], "top": "insufficient", "option_logits": null, "prompt_sha256": "2b0f13909e189564a5e5099e9094600d4804ae6ef7ef59e7435f85302977a0c0", "input_tokens": 202, "server_ms": 34.56}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.12230804376304, "probs": [0.1638594512396548, 0.04974899459992729, 0.7863915541604178], "top": "prohibited", "option_logits": null, "prompt_sha256": "a4ffa08e79c520845fb80c116412fe4c48304f5f003b514bbcbea0a7c61cfe7b", "input_tokens": 204, "server_ms": 34.61}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.84355295263231, "probs": [0.06162034871506134, 0.8591822512326693, 0.07919740005226927], "top": "insufficient", "option_logits": null, "prompt_sha256": "f51b6f4e075e67f76ff574d496ba2f9c19cc2ca38a2f56d989791df67251d344", "input_tokens": 214, "server_ms": 34.33}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.06784709356725, "probs": [0.7621344563262837, 0.23138990123980285, 0.006475642433913465], "top": "insufficient", "option_logits": null, "prompt_sha256": "baf1674ea5ff54f508700c815711c06e9a50808663840042c147d883c72021ed", "input_tokens": 214, "server_ms": 34.56}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.64665203541517, "probs": [0.14398191537109406, 0.021923360098142518, 0.8340947245307634], "top": "insufficient", "option_logits": null, "prompt_sha256": "8854be3865426f818cb7189838d63dc61b9b64da67d13087c8e6ccc2a708a8c0", "input_tokens": 213, "server_ms": 34.1}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.215728079900146, "probs": [0.2129289153471605, 0.545676239155193, 0.24139484549764645], "top": "permitted", "option_logits": null, "prompt_sha256": "913468ed99926586c9292a6dabcee9b083a7b1b1c719b6bad31fda7f7f045a2e", "input_tokens": 212, "server_ms": 34.73}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.676913034170866, "probs": [0.15943768499721103, 0.025848589203191098, 0.814713725799598], "top": "insufficient", "option_logits": null, "prompt_sha256": "acfb724b1ffcab3d0820c0a1f4cdb189396280c1faf4e622ce1f306cad1bf45e", "input_tokens": 216, "server_ms": 34.16}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.085916148498654, "probs": [0.9471510047644319, 0.04112297491157286, 0.011726020323995183], "top": "insufficient", "option_logits": null, "prompt_sha256": "7a9df7814cddba377022c022111d998db3d8a2ff48c109eb7f1aac0f5c151319", "input_tokens": 216, "server_ms": 34.57}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.89005505293608, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "ecd27e22648cba485966b925a1b12fd5a888331b66634b0f8919798224cf8072", "input_tokens": 208, "server_ms": 34.37}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.348940037190914, "probs": [0.2729138208779564, 0.3294318709588002, 0.3976543081632435], "top": "insufficient", "option_logits": null, "prompt_sha256": "86759c52a7cc961194391587ca2df8d3998c59ec236637e4acc083f151ad5242", "input_tokens": 215, "server_ms": 34.78}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.823054077103734, "probs": [0.5472741025611921, 0.331307576591957, 0.12141832084685092], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ee81091d38ca686274af8a9057ae962583e07c84d4a7c8913246f20a9adc153", "input_tokens": 209, "server_ms": 34.16}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.082756869494915, "probs": [0.10569064817366915, 0.056437743537509265, 0.8378716082888217], "top": "insufficient", "option_logits": null, "prompt_sha256": "d890e328d92ed0294fd0ca0f45891d715f1b53fae8d48848ee13c936721cc9f2", "input_tokens": 212, "server_ms": 34.38}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.88425407744944, "probs": [0.745864682612535, 0.0779433517872937, 0.17619196560017134], "top": "permitted", "option_logits": null, "prompt_sha256": "866975030f9845962316982b48126dfbaa42cd89b8d3719305c5c1cd2a7ce35c", "input_tokens": 209, "server_ms": 34.21}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.19308786094189, "probs": [0.30671712087126574, 0.5394587819774863, 0.153824097151248], "top": "permitted", "option_logits": null, "prompt_sha256": "33ed49b156b02a8251a71c89906619f22d3cff122755a441bc701809f452ec03", "input_tokens": 210, "server_ms": 34.56}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.12631715461612, "probs": [0.9411304198850982, 0.03837677130689544, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "26a5967a0d4f86a5e648fdf739a14ae8463a48c7dc0319e4b9026afcfc70ce49", "input_tokens": 200, "server_ms": 34.48}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.21683891490102, "probs": [0.2191677303480843, 0.5616645393038313, 0.2191677303480843], "top": "insufficient", "option_logits": null, "prompt_sha256": "5cb564ef00f9a81eb96ac41fe01780b4b9ffdcb60d7bc8217b9d89860c279ba3", "input_tokens": 201, "server_ms": 34.55}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.06779703497887, "probs": [0.8246471790493917, 0.16138164146161363, 0.013971179488994609], "top": "insufficient", "option_logits": null, "prompt_sha256": "e747a01e40c9029fd1004867b0db224405c02f421c64f74cd748920300cb2000", "input_tokens": 192, "server_ms": 34.59}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.03592601232231, "probs": [0.9136677967042812, 0.07428832300095413, 0.012043880294764588], "top": "B", "option_logits": null, "prompt_sha256": "3d5649d07c298093a047c4d77967454e627c1fd4792f41873c4640c036a163cd", "input_tokens": 192, "server_ms": 34.56}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.79317398928106, "probs": [0.807231387784515, 0.17909249095841714, 0.013676121257067941], "top": "insufficient", "option_logits": null, "prompt_sha256": "88ac81fbf23c9684447ca373863f9cf4c54ca0bd35eb3e960680db4187456d96", "input_tokens": 208, "server_ms": 34.32}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.13940689340234, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "insufficient", "option_logits": null, "prompt_sha256": "7263692449eeef173103012af0a80bd27db0a89dc35bf50d4fed73b80a29f045", "input_tokens": 211, "server_ms": 34.63}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.88157512806356, "probs": [0.046119873005046885, 0.9369780144263187, 0.016902112568634386], "top": "B", "option_logits": null, "prompt_sha256": "4c2c0840c1ad7e181d30f10eb62a4618c3195e936034bc7274fe3c8fc945d786", "input_tokens": 200, "server_ms": 34.37}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.13695681653917, "probs": [0.6742384798146065, 0.29826788550402233, 0.02749363468137107], "top": "B", "option_logits": null, "prompt_sha256": "2d0f22653ae25b1b9ad253802809222e0554c7bd42292a5a1637a04128e5c52f", "input_tokens": 190, "server_ms": 34.67}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.99854598380625, "probs": [0.8763745016908776, 0.09751141040687612, 0.026114087902246324], "top": "insufficient", "option_logits": null, "prompt_sha256": "676f3a162c1d11c22fd5f07e5b57e6b6e6d3a4aae515a7c2965aeac124b616dd", "input_tokens": 203, "server_ms": 34.45}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.888554925099015, "probs": [0.13968031972950484, 0.23073250176366764, 0.6295871785068274], "top": "insufficient", "option_logits": null, "prompt_sha256": "109526bdb944267ab212d1b8ba2f77a3a6c7c147016cf89ae5bc2e0e8e661adc", "input_tokens": 206, "server_ms": 34.4}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.04133606515825, "probs": [0.9673249441344611, 0.025425185481267775, 0.007249870384270957], "top": "insufficient", "option_logits": null, "prompt_sha256": "c4889b691603599fd8dca223a645f5e88b7248a364a9bef64c37a8a9e3ced5eb", "input_tokens": 197, "server_ms": 34.55}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.224018946290016, "probs": [0.49248819366063284, 0.16951261805505752, 0.3379991882843097], "top": "insufficient", "option_logits": null, "prompt_sha256": "5deb35c130c31229df54964339f9faccffa08f3ed2eb38730d86ac9ccd510aa3", "input_tokens": 192, "server_ms": 34.7}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.71655198559165, "probs": [0.1558415388693792, 0.330859461294187, 0.5132989998364338], "top": "insufficient", "option_logits": null, "prompt_sha256": "7c27c7c3361bd04c584a975218ab6b610dcab0f9bf6078cf3870b470410c383c", "input_tokens": 203, "server_ms": 34.18}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.122106878086925, "probs": [0.30165258046080173, 0.13344430167683133, 0.5649031178623669], "top": "B", "option_logits": null, "prompt_sha256": "27dba9d4c38e8b61907893c325f5851e4726ba89fa7ac9e5563a14abe89239b9", "input_tokens": 203, "server_ms": 34.63}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.791404010728, "probs": [0.9480355945752582, 0.038658346366163006, 0.013306059058578745], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f3dd4d0a864d976d5c35a7b088b3aed4b55500043acc8701dd4d35729125db8", "input_tokens": 193, "server_ms": 34.31}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.19082800671458, "probs": [0.22389674812152066, 0.7374540645868362, 0.03864918729164314], "top": "A", "option_logits": null, "prompt_sha256": "164c8915823587dee72ef8800598282728e580a02992fe7a8d8fa557ead4e3b9", "input_tokens": 192, "server_ms": 34.65}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.003136007115245, "probs": [0.9379420530362722, 0.04616732447007806, 0.01589062249364978], "top": "insufficient", "option_logits": null, "prompt_sha256": "d7c03673e95db552913947e72213d55f0938d187ca0d4fd69182cbe141bfda65", "input_tokens": 208, "server_ms": 34.35}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.15844685211778, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "A", "option_logits": null, "prompt_sha256": "b54ee64662a21acf112153e17512f10436c1938df35ab9ad182be1ff0c47c3e0", "input_tokens": 206, "server_ms": 34.44}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.157858023419976, "probs": [0.13579878113602123, 0.8376256951826758, 0.02657552368130303], "top": "A", "option_logits": null, "prompt_sha256": "8aa1b7f158bf57460e6f9f057913e0d01d913f16998cc0a46327e8aa6c93e838", "input_tokens": 200, "server_ms": 34.55}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.42071194574237, "probs": [0.25821814633948337, 0.25821814633948337, 0.4835637073210332], "top": "A", "option_logits": null, "prompt_sha256": "b17c06e9ca0e91accd4cfcb4667c5c3a42687d3209e1ecd98910eb1343a18dc2", "input_tokens": 193, "server_ms": 34.7}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.91124403849244, "probs": [0.12660014777476133, 0.09251256875552054, 0.7808872834697181], "top": "insufficient", "option_logits": null, "prompt_sha256": "324bb2046ede92322b8175d6af2961ad208eba5f52712692465ef944de2193a4", "input_tokens": 206, "server_ms": 34.25}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.335600005462766, "probs": [0.9299397393693557, 0.06263931303365462, 0.007420947596989686], "top": "insufficient", "option_logits": null, "prompt_sha256": "86e7d75bb9b41e2d9de272239185c5cce5c19dc1aac6573a737c31d47ab0563f", "input_tokens": 209, "server_ms": 34.7}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.07154607400298, "probs": [0.37290349783252347, 0.01111172675798736, 0.6159847754094891], "top": "insufficient", "option_logits": null, "prompt_sha256": "a1a07677dc816ad01403f9cff2867d5bfdaff2c29d8a4c2293ce0662a24f33ae", "input_tokens": 194, "server_ms": 34.39}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.177748046815395, "probs": [0.475785940079552, 0.1976777825029813, 0.3265362774174668], "top": "insufficient", "option_logits": null, "prompt_sha256": "82b8973ae0b97456180d256ef08822a11d20365d8039148ed863e9bb75c9bda4", "input_tokens": 192, "server_ms": 34.54}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.086677039042115, "probs": [0.14585095569321507, 0.009226876843769601, 0.8449221674630154], "top": "insufficient", "option_logits": null, "prompt_sha256": "9a672222518ab31da9c3ab2072849ef03b511e8e9d23762f89d33a83feca6ec1", "input_tokens": 216, "server_ms": 34.39}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.23476896993816, "probs": [0.09629593898139954, 0.7633942252849403, 0.14030983573366013], "top": "contradicted", "option_logits": null, "prompt_sha256": "5d2348206184a53d2da714668c8004522fc660c04abc09a4675170278b6d0ce8", "input_tokens": 214, "server_ms": 34.57}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.053456995636225, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "497e689287006b51bd4f178ebc9023050c338fca8abf790eef2112975de7f243", "input_tokens": 209, "server_ms": 34.4}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.25142800249159, "probs": [0.9369780144263187, 0.046119873005046885, 0.016902112568634386], "top": "supported", "option_logits": null, "prompt_sha256": "984a11157ce42cd37ec6209d96b53bf0d293025d6b69161f1a358f9bcc9d57db", "input_tokens": 207, "server_ms": 34.6}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.92340478301048, "probs": [0.6452505747343799, 0.2680863307491953, 0.08666309451642484], "top": "permitted", "option_logits": null, "prompt_sha256": "ea5a71f7bf051b93ca522c5ff3caef6ffdfcb5011f24af51a404b10524b250cb", "input_tokens": 228, "server_ms": 34.2}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.37190109677613, "probs": [0.43020843462499875, 0.1903144997976119, 0.3794770655773894], "top": "permitted", "option_logits": null, "prompt_sha256": "b5ac43b527f17cc117db3df096bff98320c5a248422c7b513eedeba99e619731", "input_tokens": 229, "server_ms": 34.69}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.76335304044187, "probs": [0.056437743537509265, 0.8378716082888217, 0.10569064817366915], "top": "insufficient", "option_logits": null, "prompt_sha256": "be77461c7c49b38137b436f74bcc533cf9fdcc1fbbcabe3acbe8c8c37f2272bd", "input_tokens": 222, "server_ms": 34.1}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.137707928195596, "probs": [0.1110623891201585, 0.008481107191351438, 0.88045650368849], "top": "permitted", "option_logits": null, "prompt_sha256": "12c28c53461194e05d3490b99aecb157c1965b942f5bfee5ef8163543d2bb1d0", "input_tokens": 221, "server_ms": 34.46}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.139806896448135, "probs": [0.7598853370160624, 0.23070704972212444, 0.009407613261813141], "top": "insufficient", "option_logits": null, "prompt_sha256": "0b7bc0317c30767061abcd7c39f9b71942a3c49b230cf334d1e6db7873b4dc61", "input_tokens": 216, "server_ms": 34.46}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.2179988771677, "probs": [0.9718671022270775, 0.019875215614779472, 0.008257682158143088], "top": "B", "option_logits": null, "prompt_sha256": "282f482d7bf4f65f4cc0f8dd6ca05d597f32f80783d49b010173ef79b130ffc5", "input_tokens": 210, "server_ms": 34.61}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.89859388396144, "probs": [0.08284327606718382, 0.0730741626697933, 0.8440825612630228], "top": "A", "option_logits": null, "prompt_sha256": "6bf8f97ede01d029938070b85d9e3aa545cbbbb84007fb406100f0a0a8ffa474", "input_tokens": 209, "server_ms": 34.25}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.05657692626119, "probs": [0.1753450388599366, 0.7903403209860906, 0.03431464015397285], "top": "A", "option_logits": null, "prompt_sha256": "33dc6c8523ece68e76db4eaffbb8a8470964368747977448381041503e3ad36c", "input_tokens": 211, "server_ms": 34.44}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.16525807976723, "probs": [0.03629282833642113, 0.01605512937357001, 0.9476520422900088], "top": "supported", "option_logits": null, "prompt_sha256": "6e4467088790119e033569f3066af7eedf2f72c934657e3a20082f685de045f2", "input_tokens": 226, "server_ms": 34.57}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.248848006129265, "probs": [0.044616056760944024, 0.6623675120954924, 0.2930164311435636], "top": "insufficient", "option_logits": null, "prompt_sha256": "1a718aab5e212abb4604d4e57ad3d7fb8cba756be66eba7d483eec463cb18c6b", "input_tokens": 227, "server_ms": 34.68}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.053906824439764, "probs": [0.04013769079687701, 0.03540454160362492, 0.924457767599498], "top": "contradicted", "option_logits": null, "prompt_sha256": "bafd2feb34ff693920168afa57cc5bc050eea256c82f2e9e071467805a7857f4", "input_tokens": 202, "server_ms": 34.46}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.17854805290699, "probs": [0.21057073282474073, 0.0031468025509083547, 0.7862824646243509], "top": "supported", "option_logits": null, "prompt_sha256": "c4c6394f10ee2df26d3dac75e02000ae56f28719ec24b10add7af6e77f5a4889", "input_tokens": 202, "server_ms": 34.56}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.04769583418965, "probs": [0.10502328955939726, 0.2865713245281423, 0.6084053859124604], "top": "permitted", "option_logits": null, "prompt_sha256": "37bd9a494e79ba1e91bbf2a68704609c66421b3ec11ab8a5bdd36908b17135b1", "input_tokens": 233, "server_ms": 34.41}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.28953981772065, "probs": [0.2680863307491953, 0.08666309451642484, 0.6452505747343799], "top": "insufficient", "option_logits": null, "prompt_sha256": "a115c9f1aaee5b00fa1fd3be52a599afa7112fdcf9eec8871ae98b3d6ef2183f", "input_tokens": 231, "server_ms": 34.63}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.858303938061, "probs": [0.07184163899867897, 0.8298456430479688, 0.09831271795335216], "top": "prohibited", "option_logits": null, "prompt_sha256": "100e5f7a276a412d51e2ddf940777a1ff109ea7803a8291d638760c392064d80", "input_tokens": 216, "server_ms": 34.21}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.58627315983176, "probs": [0.7469915530638082, 0.14618461002062086, 0.106823836915571], "top": "prohibited", "option_logits": null, "prompt_sha256": "6486b8dc13e88f4d5e489f3b3b76c6bb5aeed63f19005132e10380f0181726aa", "input_tokens": 223, "server_ms": 34.93}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.85957402922213, "probs": [0.8638851317960057, 0.051328051461926705, 0.08478681674206763], "top": "B", "option_logits": null, "prompt_sha256": "941cd3d853797aa1a92c8261e9339af72809062c590b26ef2db8bdbbc5d68191", "input_tokens": 222, "server_ms": 34.2}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.415970815345645, "probs": [0.046253998446132885, 0.9397029314342114, 0.014043070119655722], "top": "A", "option_logits": null, "prompt_sha256": "f9c6ae0da8d46bec4df2b4102f3aebb45748add11af2d63a981d879fea6a717c", "input_tokens": 222, "server_ms": 34.73}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.73108294606209, "probs": [0.04067015270682889, 0.7760155022030047, 0.1833143450901663], "top": "insufficient", "option_logits": null, "prompt_sha256": "4df509501a3fa1e95f377f6b2cd7ff2737ebe52dd6e95472a1a35d0fc43e3fdf", "input_tokens": 215, "server_ms": 34.16}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.48367098160088, "probs": [0.05543534738188995, 0.9330136575005004, 0.01155099511760962], "top": "A", "option_logits": null, "prompt_sha256": "d68329bad204407dddf0266c2d55c897537804b87978786e2c29649197957a18", "input_tokens": 201, "server_ms": 34.77}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.062957184389234, "probs": [0.06878691906304032, 0.03042978117418096, 0.9007832997627787], "top": "supported", "option_logits": null, "prompt_sha256": "3fdb716c314c67c93413daa3cd5a38b13bec265f706d90a97c43e5a108104f13", "input_tokens": 223, "server_ms": 34.46}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.24211920052767, "probs": [0.06893683236028562, 0.08321304657583695, 0.8478501210638775], "top": "contradicted", "option_logits": null, "prompt_sha256": "d68195061969b189d71df192da0191d4d56877cf1d8ce0d4479bc8898c998c94", "input_tokens": 219, "server_ms": 34.61}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.873974138870835, "probs": [0.016115540131028547, 0.8390475816237557, 0.14483687824521582], "top": "insufficient", "option_logits": null, "prompt_sha256": "6edd6bca63afcb6fb6d263e3e248e24e556fde2b01d965155729342ef37e411e", "input_tokens": 220, "server_ms": 34.15}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.16873796656728, "probs": [0.2862501142494634, 0.02478133063598072, 0.6889685551145559], "top": "contradicted", "option_logits": null, "prompt_sha256": "d113705714c5c456d527c8466aeda96099d32a8069c13bf0f40b1ff3ed43ad11", "input_tokens": 218, "server_ms": 34.58}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.9465840049088, "probs": [0.1609098519153101, 0.7722359149508133, 0.06685423313387646], "top": "insufficient", "option_logits": null, "prompt_sha256": "19fd16742b6e54b9e2452e09592de10d5fc44c96cab1630e5f7651b1af77d898", "input_tokens": 224, "server_ms": 34.3}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.089018151164055, "probs": [0.3043910694191852, 0.39121786116162954, 0.3043910694191852], "top": "insufficient", "option_logits": null, "prompt_sha256": "ea512df6fc353426e37889a53cae2118b9e832157a4af067cb4a85489a5c83ff", "input_tokens": 221, "server_ms": 34.41}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 36.91642405465245, "probs": [0.08097077759111387, 0.04060828666999563, 0.8784209357388906], "top": "prohibited", "option_logits": null, "prompt_sha256": "71ad1455eaea177dccc5d9fc4078b19378e2ce1c1d27289d9120a1c445349487", "input_tokens": 221, "server_ms": 34.34}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 36.866994109004736, "probs": [0.8979988326296015, 0.06048782007179681, 0.041513347298601795], "top": "permitted", "option_logits": null, "prompt_sha256": "31939a3c71e268cd4751d57ccd6a07ff44f1ca1f81c7c2f7bf165ea50668fe0a", "input_tokens": 220, "server_ms": 34.26}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.13070787489414, "probs": [0.9821934763214742, 0.008345422472445314, 0.009461101206080565], "top": "B", "option_logits": null, "prompt_sha256": "13f4ee2ce11c253ccf784222a18f900b57830384e577f4adf0030d9893ed01a9", "input_tokens": 227, "server_ms": 34.56}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.93371522240341, "probs": [0.9584447268648851, 0.039082802533792566, 0.0024724706013222815], "top": "insufficient", "option_logits": null, "prompt_sha256": "c8a6bf0c918df98324f40d3c989bf2d3593c8114ebaba9510a0de7a63d90c43f", "input_tokens": 230, "server_ms": 34.36}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.56995086930692, "probs": [0.8152011530118691, 0.1808606650321593, 0.003938181955971634], "top": "insufficient", "option_logits": null, "prompt_sha256": "d512a04940165a8dc92bcf5e8a905fa84148aca8078cba93d34897ca73ebe5fc", "input_tokens": 223, "server_ms": 34.09}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.01352491043508, "probs": [0.9509567466006987, 0.028336479497072638, 0.020706773902228682], "top": "A", "option_logits": null, "prompt_sha256": "726c7f274cbd758a0855ced1b8e06e30b0888e289eeb459d087a868fb9610ef5", "input_tokens": 210, "server_ms": 34.57}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.83812520466745, "probs": [0.03787087428859695, 0.033405034339472804, 0.9287240913719302], "top": "supported", "option_logits": null, "prompt_sha256": "0032df826a7e49e7be6e1c8d5f96ece8822a858eeb6f5280c55a79fe2e628bbb", "input_tokens": 227, "server_ms": 34.29}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.14827704243362, "probs": [0.04458832792770586, 0.7990410839770169, 0.15637058809527726], "top": "insufficient", "option_logits": null, "prompt_sha256": "c29cf273a3d78467e10369969a81a42df358b37cb972ad44f78d455f8785b7e8", "input_tokens": 224, "server_ms": 34.69}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.686151986941695, "probs": [0.6079293570442296, 0.046423581791889615, 0.34564706116388083], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b9e9fd277a7252eb95c0dea4201d1d75394e8dd03ee09f576f6fc0355881f3", "input_tokens": 220, "server_ms": 34.21}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 36.88733512535691, "probs": [0.07421350543321283, 0.7101726109045401, 0.2156138836622472], "top": "contradicted", "option_logits": null, "prompt_sha256": "25fc9b80402930a472ddf5fbb9527f22c905747ffefc59e4865da0bcf422e5bb", "input_tokens": 208, "server_ms": 34.44}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.87865403480828, "probs": [0.3596689227288795, 0.5240624893078483, 0.11626858796327226], "top": "prohibited", "option_logits": null, "prompt_sha256": "1a9834724cc432424395c6e3e4989cae7698e00f85fedac3d9a8cd1a698cd265", "input_tokens": 227, "server_ms": 34.33}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.33333991840482, "probs": [0.17244943152985545, 0.1836151222987057, 0.6439354461714388], "top": "permitted", "option_logits": null, "prompt_sha256": "cc59abf9dfd3346910e8d93dc78fe4cd21d0c20962370b801e609b82712c7ea7", "input_tokens": 228, "server_ms": 34.69}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.87125397846103, "probs": [0.11048645425133781, 0.06688603847853027, 0.8226275072701319], "top": "insufficient", "option_logits": null, "prompt_sha256": "45e75124dce70ee83cbbd2df43d7c5f3045b0da83559f8b4772e8cb6fa82dbb0", "input_tokens": 225, "server_ms": 34.23}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.471982184797525, "probs": [0.07600853330506088, 0.8779775900702179, 0.046013876624721124], "top": "permitted", "option_logits": null, "prompt_sha256": "902f5bf4dd52878d854ff660af610beb5cd00553cb6cf9fa5c1e7a5ea010391b", "input_tokens": 218, "server_ms": 34.77}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.96144511923194, "probs": [0.07014840077352269, 0.16883849753435962, 0.7610131016921177], "top": "B", "option_logits": null, "prompt_sha256": "d4a4f3c22ce6befbff78058a364e4617bda379057c95b5bacf2410a162ea27f7", "input_tokens": 221, "server_ms": 34.37}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.96276596747339, "probs": [0.7195969840802272, 0.2637194956142592, 0.016683520305513583], "top": "insufficient", "option_logits": null, "prompt_sha256": "fc0371a2d78013d514b134511d38b37de6e71a39ad2fa9b95b0119945db7a316", "input_tokens": 227, "server_ms": 34.38}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.878474056720734, "probs": [0.6328238128438389, 0.029254643239392513, 0.33792154391676854], "top": "A", "option_logits": null, "prompt_sha256": "5521ebbd2304bdf6049132465a2b8c7b28c862f462f50f98d90efeb483601e64", "input_tokens": 207, "server_ms": 34.32}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.087426986545324, "probs": [0.22351946247571317, 0.12708523693891682, 0.64939530058537], "top": "A", "option_logits": null, "prompt_sha256": "c11ad01ccd57e2b3b849c07b89887124412bce4a8e5506adaeb954f23c598ac7", "input_tokens": 197, "server_ms": 34.46}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 36.88112506642938, "probs": [0.038658346366163006, 0.013306059058578745, 0.9480355945752582], "top": "supported", "option_logits": null, "prompt_sha256": "8d5e868c2065409c5dcedc10c1d875c102dfbb48d2433b234f0ad0456f1049bc", "input_tokens": 213, "server_ms": 34.28}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.1362071018666, "probs": [0.01953747345276732, 0.9553520377742281, 0.025110488773004678], "top": "contradicted", "option_logits": null, "prompt_sha256": "6ac7738979b2b31bef2f7e2f3d8984b0b2b559ee71fa4d2b728e28884e3183b5", "input_tokens": 212, "server_ms": 34.55}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 37.059787195175886, "probs": [0.05100292527189731, 0.035003776957469146, 0.9139932977706335], "top": "contradicted", "option_logits": null, "prompt_sha256": "66c4b4c8b3dc52b623b51acf50a373bf625ad5559da7c3bf90c47f56e7c2829b", "input_tokens": 202, "server_ms": 34.44}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.40518097765744, "probs": [0.09842495451008554, 0.01699017293024806, 0.8845848725596664], "top": "insufficient", "option_logits": null, "prompt_sha256": "7db376c4d8c96d2c69fda861429911890ec2d70b0f375e60f65c846f1693e18a", "input_tokens": 203, "server_ms": 34.81}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.83599294163287, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "03449c2d41ee64f00d22b4b6e619aa69ce6d623d00b4972f503f5ed336bba08e", "input_tokens": 218, "server_ms": 34.15}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.275690119713545, "probs": [0.9331701885712249, 0.04890646327936172, 0.017923348149413328], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ed1d2ff423bdb9e622336508299a3180255754a84d1847b9b5ba5d1d5623692", "input_tokens": 221, "server_ms": 34.73}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.06030594184995, "probs": [0.05755147392994642, 0.0327217252711575, 0.9097268007988961], "top": "insufficient", "option_logits": null, "prompt_sha256": "fb0073f867c77a04f9287adbd88276517a2a85e5cf9dcfb04c81ff5ba768e03f", "input_tokens": 215, "server_ms": 34.44}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.17554803006351, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "insufficient", "option_logits": null, "prompt_sha256": "51c4ac799c31ceb752046516252cb8d59c3a3c1654c685c82476530a4d74a689", "input_tokens": 220, "server_ms": 34.6}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.574610974639654, "probs": [0.040967527214491276, 0.8323022347202245, 0.12673023806528433], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a5a9b08e7ca06d1f5808cdcd3270c79b064cd4fed16b4be278fc7d5f58a1d1f", "input_tokens": 206, "server_ms": 34.02}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.30502910912037, "probs": [0.048847636440916636, 0.07578275550521898, 0.8753696080538644], "top": "insufficient", "option_logits": null, "prompt_sha256": "555fd1e019506f048448ae23a6bac4fe7f3e9f2fe60181be02acfdf8225318e5", "input_tokens": 212, "server_ms": 34.7}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.66696301661432, "probs": [0.07319171247064139, 0.2264135372721482, 0.7003947502572104], "top": "insufficient", "option_logits": null, "prompt_sha256": "1fc713f5dfc86b0d90f75094b087d920fbfeb9830fb4eec74f78d2f1d67fbf66", "input_tokens": 203, "server_ms": 34.1}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 37.286919075995684, "probs": [0.9387496456243781, 0.055776151759141294, 0.005474202616480596], "top": "B", "option_logits": null, "prompt_sha256": "77e6633b56fd111f3cf38ff30d1c308c084b3de608efb9682cf02409dc31c9db", "input_tokens": 191, "server_ms": 34.69}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.889794981107116, "probs": [0.46903312718147117, 0.38856479455009935, 0.14240207826842943], "top": "supported", "option_logits": null, "prompt_sha256": "83104242d8900e7e51e6ca6f99d29588121dd7153c0fda9d487e3eafef91196e", "input_tokens": 224, "server_ms": 34.29}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.33570105396211, "probs": [0.7720397533404616, 0.11755438599864407, 0.11040586066089443], "top": "insufficient", "option_logits": null, "prompt_sha256": "66f0b52c1eee120f16069ed75fbd7bfc556b5a3dc09c27f809799b32ae74e959", "input_tokens": 221, "server_ms": 34.71}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.113475846126676, "probs": [0.014236196905258059, 0.6140385266223682, 0.3717252764723737], "top": "insufficient", "option_logits": null, "prompt_sha256": "fac76f29223ea6573fb84bb8cb70a27292765e7f4f90ebb19001e8a0b7bf0533", "input_tokens": 229, "server_ms": 34.52}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 37.13076701387763, "probs": [0.06291220787817095, 0.0030966605414949147, 0.9339911315803341], "top": "supported", "option_logits": null, "prompt_sha256": "89df7654ae643d961d5bd0a1cf472717b674fe14a8f5377a6918c1103303e14f", "input_tokens": 226, "server_ms": 34.53}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.14467701502144, "probs": [0.5216500126916775, 0.26161676918070537, 0.21673321812761712], "top": "prohibited", "option_logits": null, "prompt_sha256": "73a309dd6a1605e1d928e2a4947516f3ae54fda513d77cea9d4fdf892dbbebb5", "input_tokens": 226, "server_ms": 34.51}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 37.416951963678, "probs": [0.843310768581531, 0.1455727910071266, 0.01111644041134243], "top": "prohibited", "option_logits": null, "prompt_sha256": "97510bceb98f70589de7351d3f810c2e4c175b03bbf6ddec55eca00ef9c199ee", "input_tokens": 215, "server_ms": 34.81}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.89853404648602, "probs": [0.42929779219364844, 0.486689574582582, 0.08401263322376958], "top": "permitted", "option_logits": null, "prompt_sha256": "5a1d1ca84275c476cd245cafcd0e1fd36598a6dae3c643b8ea9d60c5b57d6535", "input_tokens": 225, "server_ms": 34.28}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 37.38109115511179, "probs": [0.19560247361171754, 0.23611000151056785, 0.5682875248777147], "top": "permitted", "option_logits": null, "prompt_sha256": "fec6fd7a8c54ee7856b56b8cc85aa3ff4cb617820fd52f7faf204e2efb4a86ff", "input_tokens": 224, "server_ms": 34.83}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 36.941925063729286, "probs": [0.9323898466010938, 0.02450695094155103, 0.04310320245735524], "top": "B", "option_logits": null, "prompt_sha256": "fcc6cca56205c973fe5ee42de7bbca225406234b5346f63e55af7d8c5f115d9a", "input_tokens": 213, "server_ms": 34.23}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 37.31670998968184, "probs": [0.643833389199536, 0.3438005390976864, 0.012366071702777563], "top": "insufficient", "option_logits": null, "prompt_sha256": "176f3f577d5802518ba6bb8a36039f8c2c7fa4991aac2e236870e0189d0b31a6", "input_tokens": 218, "server_ms": 34.62}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 36.94967506453395, "probs": [0.020042268594912593, 0.21743089555343206, 0.7625268358516554], "top": "B", "option_logits": null, "prompt_sha256": "8494413c37184f76cf07859eef5fe11f7373b1acb0da7cbd005226a5bb00610d", "input_tokens": 209, "server_ms": 34.34}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 37.151667987927794, "probs": [0.33058297459294167, 0.050336110784327995, 0.6190809146227304], "top": "A", "option_logits": null, "prompt_sha256": "70f353d85ca31feb6cd6d5c421fe7a2e877a313657d44902068dc2f6ef451bf8", "input_tokens": 207, "server_ms": 34.55}], "seconds": 5.3, "failures": 0}, "multifield": {"rows": [{"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.14720923592665844, 0.8527907640733415], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.8214121479792841, 0.17115665028167817, 0.00743120173903778], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 45.86503515020013, "probs": [0.9305404571279347, 0.06673814075012655, 0.0027214021219388934], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.8095965925134007, 0.07462670936879237, 0.11577669811780701], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.9471470189565372, 0.052852981043462664], "top": "same"}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.7940519079566, "probs": [0.9350414422025257, 0.05915293776423124, 0.005805620033243134], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.22187782400103293, 0.778122175998967], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.09874410354776683, 0.06776898998409085, 0.8334869064681424], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.07967467169628852, 0.9203253283037116], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.55907898955047, "probs": [0.10841793826059654, 0.0843556864709907, 0.8072263752684127], "top": "tavern"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.21123545993246395, 0.788764540067536], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.11796301878836102, 0.8782943442297925, 0.0037426369818465326], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.8527907640733415, 0.14720923592665844], "top": "same"}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.78143202140927, "probs": [0.24350964936246988, 0.7532804299591004, 0.0032099206784296486], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.58771995268762, "probs": [0.9344875966425453, 0.06294564908924384, 0.002566754268210852], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.7318060856600669, 0.2681939143399331], "top": "moved"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.727805131100519, 0.25050783114702185, 0.02168703775245924], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere"}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.668110786005855, "probs": [0.8387192579214904, 0.15415437985672864, 0.007126362221781067], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.10592235395880453, 0.8940776460411956], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.928892622801621, 0.06661995929549502, 0.004487417902884053], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same"}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.40141818486154, "probs": [0.9588514552888884, 0.039099387288032916, 0.002049157423078645], "top": "stay"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.14648079183179633, 0.8485708719675852, 0.004948336200618522], "top": "meet_wren"}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.9248060506082927, 0.07519394939170732], "top": "same"}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 44.99178403057158, "probs": [0.24334788832895243, 0.7527800186067107, 0.0038720930643368974], "top": "meet_wren"}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds"}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same"}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.0275089610368, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.8089141677995819, 0.1910858322004182], "top": "stay"}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.95355093665421, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.9439168356016766, 0.0560831643983235], "top": "same"}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.226144164800644, "probs": [0.9156019658234928, 0.08439803417650704], "top": "stay"}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.3201997500236374, 0.6798002499763626], "top": "holds"}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.12511350509772579, 0.8748864949022742], "top": "departure"}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 47.94858116656542, "probs": [0.13214384255458153, 0.8678561574454184], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.25606104458540857, 0.7439389554145914], "top": "holds"}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.08439803417650704, 0.9156019658234928], "top": "departure"}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.11840617139815589, 0.8815938286018441], "top": "elsewhere"}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.35686506703496, "probs": [0.10012611628383591, 0.8998738837161642], "top": "departure"}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.13214384255458153, 0.8678561574454184], "top": "holds"}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.8604937671763584, 0.1395062328236417], "top": "stay"}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.9502009729447547, 0.04979902705524531], "top": "same"}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.568967962637544, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.8184242551251981, 0.18157574487480185], "top": "stay"}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.307005083188415, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.12511350509772579, 0.8748864949022742], "top": "holds"}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.9248060506082927, 0.07519394939170732], "top": "stay"}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.9558139322179339, 0.04418606778206604], "top": "same"}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64688916131854, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.7193152314525416, 0.2806847685474584], "top": "moved"}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.08937422170979005, 0.91062577829021], "top": "outside"}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.2863849028945, "probs": [0.07519394939170732, 0.9248060506082927], "top": "outside"}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.15526082656247686, 0.844739173437523], "top": "holds"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.8363317223730603, 0.1636682776269396], "top": "stay"}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.9405016353866033, 0.059498364613396776], "top": "same"}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.61014895141125, "probs": [0.9203253283037116, 0.07967467169628852], "top": "stay"}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.45308504361568214, 0.5469149563843178], "top": "holds"}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.25606104458540857, 0.7439389554145914], "top": "outside"}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.15526082656247686, 0.844739173437523], "top": "elsewhere"}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.64811897277832, "probs": [0.43758935794935333, 0.5624106420506467], "top": "outside"}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.1724380672551571, 0.8275619327448428], "top": "holds"}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay"}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.9203253283037116, 0.07967467169628852], "top": "same"}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.95878303796053, "probs": [0.936892368084393, 0.06310763191560703], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.18157574487480185, 0.8184242551251981], "top": "holds"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.9053865123447689, 0.09461348765523125], "top": "stay"}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.9156019658234928, 0.08439803417650704], "top": "same"}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 48.04007103666663, "probs": [0.91062577829021, 0.08937422170979005], "top": "stay"}], "seconds": 1.0, "failures": 0}}, "finished_utc": "2026-09-30T16:28:00Z"}
Reference in New Issue View Git Blame Copy Permalink