esh-pfi-infrastructure/services/intern-decision-serve/acceptance/gpu1-live/sets-live.json

{"label": "gpu1-live", "backend": "semif", "url": "http://intern-decision.fv.internal:8033", "started_utc": "2026-09-30T16:45:39Z", "conditions": {"single": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 70.16763702267781, "probs": [0.042316276858002114, 0.042316276858002114, 0.9153674462839958], "top": "contradicted", "option_logits": null, "prompt_sha256": "426dea1e774016ae24264f3548e2d92557e0498fbbbde8b49ae3037ceba07b3b", "input_tokens": 204, "server_ms": 49.74}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.991382010281086, "probs": [0.9651783660446455, 0.017410816977677268, 0.017410816977677268], "top": "supported", "option_logits": null, "prompt_sha256": "53ef2749d9b9ada139227e9cd46f40960ef0f1e1ee9b26a857e94dea6a086188", "input_tokens": 202, "server_ms": 35.43}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.430486015509814, "probs": [0.038153743171136366, 0.9356609917402224, 0.026185265088641236], "top": "supported", "option_logits": null, "prompt_sha256": "2c882bbaf28d07e029b9e32b83997ee09d96a9022542e5d9e6a23bbdf3221fca", "input_tokens": 196, "server_ms": 35.1}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.80789999617264, "probs": [0.9233581325882398, 0.031192392974493464, 0.045449474437266706], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b598e5112b699f2db4d6df47efb4be686f1cfa1a12c81ec223f0a3e87b77779", "input_tokens": 196, "server_ms": 35.29}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.322380980011076, "probs": [0.9697243649711375, 0.012782812574961349, 0.01749282245390113], "top": "supported", "option_logits": null, "prompt_sha256": "8edab8e3530019fcbb2106afd7c081fa4e1792831e8d12674dc7d17114b232aa", "input_tokens": 209, "server_ms": 35.09}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.636603977996856, "probs": [0.06541354168337463, 0.02251509853480652, 0.9120713597818187], "top": "contradicted", "option_logits": null, "prompt_sha256": "ee0c1e9d86e5505c512010f9e6eecf9335d9c53a6a14fb35afb57c1b96c6faf3", "input_tokens": 206, "server_ms": 35.24}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.20438201306388, "probs": [0.9398833253029221, 0.03832591832238971, 0.02179075637468831], "top": "contradicted", "option_logits": null, "prompt_sha256": "8e34e8add460036c4de5af761302128fc82f3c4ee8359a388a0c4c0ddfc2fbe6", "input_tokens": 208, "server_ms": 34.97}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.79257898312062, "probs": [0.04454376114064005, 0.050498708227220655, 0.9049575306321394], "top": "insufficient", "option_logits": null, "prompt_sha256": "f99c0189a62f8147477237b83279769eabf12f5a510b8633c9ec416d1a8578e4", "input_tokens": 200, "server_ms": 35.52}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.5908910096623, "probs": [0.03022311796549784, 0.017183792327615903, 0.9525930897068864], "top": "contradicted", "option_logits": null, "prompt_sha256": "058a4a2adfd935863953520b8eb055a2d568122a0e8897052a2c93a5cc70cfc4", "input_tokens": 194, "server_ms": 35.11}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.601427003741264, "probs": [0.9695832653382126, 0.010588224632471017, 0.01982851002931642], "top": "supported", "option_logits": null, "prompt_sha256": "3775c1c18dc072e2a88a320060d75f34a4b413ba692a272f2309bf2ea1fdd1ae", "input_tokens": 194, "server_ms": 35.13}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.274250007234514, "probs": [0.9770960110414373, 0.008302110383965793, 0.014601878574596829], "top": "supported", "option_logits": null, "prompt_sha256": "f39d71e8a36f2589074ebf8c9534db931a17d7325e5a08a85a26cfe652508c33", "input_tokens": 193, "server_ms": 34.98}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.962122950702906, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "insufficient", "option_logits": null, "prompt_sha256": "1dc41cdba9068a2dd371d85948acfbfce09acc0e572837804762116bd6ea30dc", "input_tokens": 192, "server_ms": 35.27}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.2721999916248, "probs": [0.014374500594547012, 0.023744679136544822, 0.9618808202689082], "top": "supported", "option_logits": null, "prompt_sha256": "3e3295b1cac557e05f098343fbac4a14b20853589fe70e742bbf9fd568508aea", "input_tokens": 205, "server_ms": 35.2}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.04409802192822, "probs": [0.04058846897644321, 0.024571355391379026, 0.9348401756321778], "top": "contradicted", "option_logits": null, "prompt_sha256": "5ee9caa002a79a1b78f0cadfdd2b0d175b0edfece980ac05df53e1cf8a42a249", "input_tokens": 205, "server_ms": 35.0}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.368213993962854, "probs": [0.03391183675453223, 0.02327400569089846, 0.9428141575545693], "top": "contradicted", "option_logits": null, "prompt_sha256": "0e56f2c135ca9571701644797b73406989c534535aa796629f128168ea33b337", "input_tokens": 199, "server_ms": 35.27}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.29708702629432, "probs": [0.9649096748354914, 0.022370979735516088, 0.012719345428992583], "top": "insufficient", "option_logits": null, "prompt_sha256": "de2a13ff9508e3815fbfa49cbd7d14277f44444077e2c2b3ed6f47582c489835", "input_tokens": 194, "server_ms": 35.46}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.1022849609144, "probs": [0.9233581325882398, 0.045449474437266706, 0.031192392974493464], "top": "contradicted", "option_logits": null, "prompt_sha256": "f9d3a29be41d2e79b7041d491ae148dddb06945d072971c57f34e82a24acf497", "input_tokens": 198, "server_ms": 35.26}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.53796004783362, "probs": [0.027959089680252548, 0.938291740342913, 0.03374916997683453], "top": "supported", "option_logits": null, "prompt_sha256": "76fed3ccb4c68d0f339fdb718ab37085a0c4f31a3626cb9b4434006be6c01d65", "input_tokens": 198, "server_ms": 35.19}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.22376999538392, "probs": [0.033806648127806344, 0.9398897419850005, 0.026303609887193], "top": "supported", "option_logits": null, "prompt_sha256": "6c738acf1c3e484397fbb45b0086bef4e5fb0fb841cb7e0caaf5b459030b22c9", "input_tokens": 194, "server_ms": 35.06}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.34067402873188, "probs": [0.043830111066642266, 0.008055858485872225, 0.9481140304474855], "top": "insufficient", "option_logits": null, "prompt_sha256": "d31b190b22fdd50db3227d978812eca7cf1d90b460f8cf4804e2788992b4bd5c", "input_tokens": 194, "server_ms": 35.31}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.49713598843664, "probs": [0.04090374672423521, 0.0169945383164516, 0.9421017149593132], "top": "supported", "option_logits": null, "prompt_sha256": "b62913ec06f42fa6b26014e6a574953154612aa6bbaa9e900ecbd21c91a58901", "input_tokens": 211, "server_ms": 35.24}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.27685398515314, "probs": [0.04194866830694898, 0.9074154634408776, 0.05063586825217336], "top": "contradicted", "option_logits": null, "prompt_sha256": "2d137ab4419274208800e3fb77890670f63f751139fc8acac92b7db7b6a730a5", "input_tokens": 211, "server_ms": 35.05}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.41646004328504, "probs": [0.03579134697254336, 0.9345577469581762, 0.029650906069280464], "top": "contradicted", "option_logits": null, "prompt_sha256": "65d98fd6662f81fa63534de6fb0b5a120a125650cf62a7de2fd2699049de80f3", "input_tokens": 201, "server_ms": 35.29}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.68917196756229, "probs": [0.9690991420237913, 0.016418485982945336, 0.014482371993263405], "top": "insufficient", "option_logits": null, "prompt_sha256": "93b403a86d3506f515c3cddbe5919eaa0281fcf4795a60184ee515c1fddfa7b1", "input_tokens": 197, "server_ms": 35.46}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.246633033268154, "probs": [0.19902597038322306, 0.6555365755036917, 0.14543745411308528], "top": "prohibited", "option_logits": null, "prompt_sha256": "e3b57d01783bd19503abd5a9abd515403f0f8aeb84a581a45eb1402c7179318b", "input_tokens": 212, "server_ms": 35.15}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.3244419968687, "probs": [0.030153846392648304, 0.9504097526239933, 0.019436400983358377], "top": "permitted", "option_logits": null, "prompt_sha256": "55acba94f1587a7f54a93f767747e852dadb1d8ba1f937c3bcb8fbbca8bd25f1", "input_tokens": 218, "server_ms": 35.29}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.68051101686433, "probs": [0.06220125487024129, 0.07051679026836535, 0.8672819548613933], "top": "permitted", "option_logits": null, "prompt_sha256": "e94c1ed6f217a9215c05c73403057cbe6d03693358147199edde077f76c0ca17", "input_tokens": 203, "server_ms": 35.22}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.55007399339229, "probs": [0.8434852515678085, 0.10639876347043403, 0.0501159849617574], "top": "insufficient", "option_logits": null, "prompt_sha256": "967a6559cc35259d1050773013767bed123a54ef95de445efc721df0c5fc11d1", "input_tokens": 209, "server_ms": 35.12}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.91803599195555, "probs": [0.8617058851120665, 0.09587928195428774, 0.042414832933645806], "top": "permitted", "option_logits": null, "prompt_sha256": "a9b6b7c8cbc195a4c73d593d73790c337af636f29e274a66e99c67970a64388c", "input_tokens": 209, "server_ms": 34.96}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.05886098742485, "probs": [0.050887592888715624, 0.9119264943284082, 0.03718591278287608], "top": "prohibited", "option_logits": null, "prompt_sha256": "ceb1ed19cf8b23ecd67e8000ee14fde3623a1ca2330dbcd10a420e8e0426871d", "input_tokens": 214, "server_ms": 35.43}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.46513800416142, "probs": [0.18515695889853595, 0.445649845141063, 0.3691931959604011], "top": "insufficient", "option_logits": null, "prompt_sha256": "695e83f16f10e2d5bfe4d413a0893af1fc3b78dacb36e19f8aa4405a1aa63348", "input_tokens": 209, "server_ms": 34.85}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.26124203950167, "probs": [0.08420898053696053, 0.8579976192216164, 0.05779340024142296], "top": "insufficient", "option_logits": null, "prompt_sha256": "fefdc3621c12fcff6225a3b6b6eb374ab324df4afc63642748f3f7c4cd063503", "input_tokens": 209, "server_ms": 35.23}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.24963997211307, "probs": [0.10672401314545814, 0.04721229751644958, 0.8460636893380923], "top": "prohibited", "option_logits": null, "prompt_sha256": "1429197ed3833935b9b2a1ab079c1e5d9044b0b29b0a78dcc163d30cfbdd5d73", "input_tokens": 208, "server_ms": 35.02}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.6466590128839, "probs": [0.876117367942713, 0.08075843891914115, 0.04312419313814581], "top": "permitted", "option_logits": null, "prompt_sha256": "ef2c7f94a66d4c5d41ea39896dadcd381ffdb44fd6b84cb7ebfb2dec29dda2d7", "input_tokens": 211, "server_ms": 35.28}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.4927980042994, "probs": [0.0477035724410535, 0.042078233664266926, 0.9102181938946795], "top": "permitted", "option_logits": null, "prompt_sha256": "1ac0c107839fcdcef568dead30d2c7ad5003790cab4a6fb5f59853045b91614b", "input_tokens": 202, "server_ms": 34.95}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.517003019806, "probs": [0.8309481792947823, 0.09245697019897443, 0.07659485050624319], "top": "insufficient", "option_logits": null, "prompt_sha256": "d14cc5fbd5c5014be587672c8a847ec7869217ec46b2986b92cb9590d1d60f62", "input_tokens": 204, "server_ms": 35.38}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.192469000350684, "probs": [0.035003776957469146, 0.05100292527189731, 0.9139932977706335], "top": "permitted", "option_logits": null, "prompt_sha256": "0411bf24ff3c7ae3c15cbd4c8e2be427e910559a0eec4c3e642f1b5155a41238", "input_tokens": 214, "server_ms": 35.02}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.26417996548116, "probs": [0.1258463317845894, 0.7762376391176652, 0.09791602909774536], "top": "prohibited", "option_logits": null, "prompt_sha256": "0ac22c9e253b2c6f15c7fd533350d31256661a3cc46f7ea6901dd6d52416aa74", "input_tokens": 214, "server_ms": 35.16}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.904386004433036, "probs": [0.8803184943057364, 0.09795024786186077, 0.021731257832402765], "top": "prohibited", "option_logits": null, "prompt_sha256": "93e7cdf8b0a9d2e91ef77f746e20b5d07f380d466b67f039b8eca81d82785f37", "input_tokens": 213, "server_ms": 35.07}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 57.23887198837474, "probs": [0.7788844976918236, 0.08666400869190902, 0.13445149361626738], "top": "insufficient", "option_logits": null, "prompt_sha256": "0cc52cc068b469c091089844eda6fdb0ce8290f343d6110dea057157fc6c159b", "input_tokens": 212, "server_ms": 35.46}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.681224981322885, "probs": [0.9170956118639788, 0.07003258404452323, 0.012871804091497973], "top": "prohibited", "option_logits": null, "prompt_sha256": "07df45fd805fb49da7cbb17c4c9821cbd45bbd4a8559d853d3f2058d3d1f8abc", "input_tokens": 216, "server_ms": 35.13}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.89592900266871, "probs": [0.046511751700819434, 0.9449395114357498, 0.008548736863430634], "top": "permitted", "option_logits": null, "prompt_sha256": "bc5166044ea70dfe1ffe698ff69c434fcd560984f8da98b811bd392c6ef2375e", "input_tokens": 216, "server_ms": 35.24}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.98835998494178, "probs": [0.081707168383676, 0.03188304278259795, 0.8864097888337261], "top": "permitted", "option_logits": null, "prompt_sha256": "d5bd19d3c77b70f8cbed83668596689125bf1db168f76b7b3a357f8846b6acc6", "input_tokens": 208, "server_ms": 35.0}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.660174016840756, "probs": [0.17703104527829333, 0.07355221007985424, 0.7494167446418524], "top": "insufficient", "option_logits": null, "prompt_sha256": "71bff3cfe089f548b584212a5839219740fe75f47f8409f1aa4e0add7ba7fad6", "input_tokens": 215, "server_ms": 35.17}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.97512303898111, "probs": [0.1288364204458939, 0.6583437011177077, 0.21281987843639838], "top": "permitted", "option_logits": null, "prompt_sha256": "6ae4ca1f7ec14a26e584776f03a32db110efc2f1df49537f9311888096dc89fa", "input_tokens": 209, "server_ms": 35.01}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.37657296331599, "probs": [0.7610131016921177, 0.16883849753435962, 0.07014840077352269], "top": "prohibited", "option_logits": null, "prompt_sha256": "7b372cc754513ce412e4325cbf032ee998cc4bd502c5f3b9a3db21891fb9b6d2", "input_tokens": 212, "server_ms": 35.2}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.02162203285843, "probs": [0.09687148818542426, 0.8176803730758856, 0.0854481387386902], "top": "prohibited", "option_logits": null, "prompt_sha256": "cc657ff3d559e698494e44382f2bf995e204490c096409ac7944aaeec100310f", "input_tokens": 209, "server_ms": 35.04}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.18939299508929, "probs": [0.12635484417148546, 0.30412045992272907, 0.5695246959057854], "top": "insufficient", "option_logits": null, "prompt_sha256": "ba5da8bcd12908f5ace1d698c11786ba073dea3bff4318245007297e559d4988", "input_tokens": 210, "server_ms": 35.24}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.47618803009391, "probs": [0.04399527306105078, 0.9516867697626097, 0.004317957176339503], "top": "A", "option_logits": null, "prompt_sha256": "02cea35cf16bc27b15251895e9b0de62fcd0662c7f918883d61aea0893118999", "input_tokens": 200, "server_ms": 35.25}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.6555690234527, "probs": [0.9583922703489581, 0.03237589671648153, 0.00923183293456039], "top": "B", "option_logits": null, "prompt_sha256": "df09a6812cdaef0b4a9af65efb2b57c9acba2e4038a34c88fa1031636c5c12d1", "input_tokens": 201, "server_ms": 35.24}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.34330297773704, "probs": [0.06149524169687701, 0.9129549413995451, 0.025549816903577924], "top": "B", "option_logits": null, "prompt_sha256": "557cfcbee9d51b363d9d810109c9f6860806c61b492f0c0f7c5a517000b80c7e", "input_tokens": 192, "server_ms": 34.81}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.929802019614726, "probs": [0.06765176949122025, 0.8859181944824366, 0.0464300360263431], "top": "A", "option_logits": null, "prompt_sha256": "f5843f569f14042388a7d34f44ca703413a459c98cbe772b6732757618159917", "input_tokens": 192, "server_ms": 35.25}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.408437001053244, "probs": [0.03607939080870998, 0.9420789483611384, 0.02184166083015174], "top": "B", "option_logits": null, "prompt_sha256": "b2b6c583e4d4cbb5eab53cbda1caad02e1d8c2de98ccae679c147a769fff0cd0", "input_tokens": 208, "server_ms": 35.0}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.69659597333521, "probs": [0.043982976135970124, 0.9514207741016708, 0.004596249762359112], "top": "A", "option_logits": null, "prompt_sha256": "2b29fd9860ebcb9c25eca46c46d974680f12177b7204d8e8aa21bdb4e410c90e", "input_tokens": 211, "server_ms": 35.18}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.92702099541202, "probs": [0.08293947217253797, 0.017281996371705088, 0.899778531455757], "top": "A", "option_logits": null, "prompt_sha256": "36e701f3ac2aeef85b1f94539bc461c0fe9692de390d199289cc4b0010d1169c", "input_tokens": 200, "server_ms": 35.12}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.621117004193366, "probs": [0.2086204876721029, 0.5692513354541036, 0.22212817687379355], "top": "A", "option_logits": null, "prompt_sha256": "e193fd9b4928c86c295aca78ce3664f3800cb887c08a75cc81f4c27d8cac6f76", "input_tokens": 190, "server_ms": 35.61}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.4305180083029, "probs": [0.052456499742245456, 0.940041937252148, 0.007501563005606651], "top": "A", "option_logits": null, "prompt_sha256": "7d78c4010db6e376515791dec2799510665f8b78b8fc125e6c81ab272a5e54a4", "input_tokens": 203, "server_ms": 35.19}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.693385038059205, "probs": [0.9333925565458442, 0.06287188741231692, 0.0037355560418387687], "top": "B", "option_logits": null, "prompt_sha256": "f7f1c449378cfeab1aa6f0d8b8956c0ebdc42fc41981715151ef635cd7c30a64", "input_tokens": 206, "server_ms": 35.27}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.116972965653986, "probs": [0.03231411729008851, 0.9565634821799458, 0.011122400529965637], "top": "B", "option_logits": null, "prompt_sha256": "2a410919fe9bcc4c3af0ee5d5cb0a18027483d7607cc67d8e2b36aac780017b6", "input_tokens": 197, "server_ms": 34.9}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.685306978877634, "probs": [0.7344499147820565, 0.23742239353898825, 0.028127691678955267], "top": "insufficient", "option_logits": null, "prompt_sha256": "55951e665a7f57802d87a47f36e9f026fa34961d6ab95756cc930b7f23c58b5e", "input_tokens": 192, "server_ms": 35.18}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.392213023267686, "probs": [0.9365053724148908, 0.05924555068392171, 0.004249076901187515], "top": "B", "option_logits": null, "prompt_sha256": "2ebe271bc5824d82d2d78bf594230128f6c575a57fd8baf2e302594e8e197a3c", "input_tokens": 203, "server_ms": 35.06}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.405229991767555, "probs": [0.9641301142276885, 0.025341212170873196, 0.010528673601438379], "top": "A", "option_logits": null, "prompt_sha256": "541ebe99be4ec526fe1a7c254e8aac9a46d20f03cf70c55182ff21d2e81c7730", "input_tokens": 203, "server_ms": 35.37}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.20586898596957, "probs": [0.044006830551321936, 0.9519367233977202, 0.004056446050957758], "top": "A", "option_logits": null, "prompt_sha256": "56f8c45e2826df48de51103c0d57592e857a2d2c5a2b4c7e44ae6a5c482fdbeb", "input_tokens": 193, "server_ms": 34.88}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.26151999179274, "probs": [0.08732560804889346, 0.07702792615673913, 0.8356464657943673], "top": "insufficient", "option_logits": null, "prompt_sha256": "2cab104825b0b30d9ff566030d11a1e3b09b321b3d38b90c0efed0cce5a7bcd6", "input_tokens": 192, "server_ms": 35.3}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.98640998080373, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "0ad946cf29b69aa1852873ef868eff6aabdae1ecf90cb02a01664377aa52c147", "input_tokens": 208, "server_ms": 34.91}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.951664038933814, "probs": [0.021415606524360562, 0.054882108682312046, 0.9237022847933274], "top": "B", "option_logits": null, "prompt_sha256": "e4debc81a5f840b3b59b78b99150b6fe7d352c9ebff0a75758e00ad238927abe", "input_tokens": 206, "server_ms": 38.18}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.485683027654886, "probs": [0.04585791045380365, 0.182323036446684, 0.7718190530995124], "top": "B", "option_logits": null, "prompt_sha256": "56a4f8a9b0eec586f7f826729bdafc23052803d1b5c00ad8560b665e05e864b0", "input_tokens": 200, "server_ms": 36.76}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.92668795725331, "probs": [0.48466933476690643, 0.030661330466187017, 0.48466933476690643], "top": "insufficient", "option_logits": null, "prompt_sha256": "b8455d3ab548338ef64f7e77be5b8dabf8a597e33d0cc63fe8193ef80e622bfb", "input_tokens": 193, "server_ms": 36.37}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.33419398125261, "probs": [0.9617904722438735, 0.03459439130569816, 0.003615136450428284], "top": "B", "option_logits": null, "prompt_sha256": "b858c6bbc8c8351f4346fc5e3d852fe6ee9b785dc6b94d360ae628024eb06709", "input_tokens": 206, "server_ms": 36.14}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.46172702452168, "probs": [0.03667902066255044, 0.9577360596916973, 0.005584919645752244], "top": "A", "option_logits": null, "prompt_sha256": "0a88a2f84514595473b1cce8e46c281ec4874b3089ad734915859ee7dedbe235", "input_tokens": 209, "server_ms": 36.22}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.8139780187048, "probs": [0.9824562054566083, 0.012163109080848325, 0.005380685462543299], "top": "A", "option_logits": null, "prompt_sha256": "2c9673b1cb4b3fbd58623553b451dc156780891cb5dcc9b6308a8e8956ad3707", "input_tokens": 194, "server_ms": 35.97}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.982704994268715, "probs": [0.6156245007589809, 0.35002223803687876, 0.03435326120414038], "top": "insufficient", "option_logits": null, "prompt_sha256": "27a13455c83f546a6536d5176d6495279bfc3e008a8be3357720f358e0b0ba51", "input_tokens": 192, "server_ms": 35.73}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.647270950023085, "probs": [0.9541450372778729, 0.02077619807256723, 0.02507876464955978], "top": "contradicted", "option_logits": null, "prompt_sha256": "cd2424143ed01cada3e3de82c5fc7baff6151e8e0c9a2a5cf3dc62da3bdb5df4", "input_tokens": 216, "server_ms": 35.53}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.142164018936455, "probs": [0.06521723973326335, 0.02544848910395107, 0.9093342711627855], "top": "supported", "option_logits": null, "prompt_sha256": "aa8d2e4b84513eefef53cabda30cc69e33d534dbf16a8b10c03c0b7a81685221", "input_tokens": 214, "server_ms": 35.34}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.44560799514875, "probs": [0.9742641243789874, 0.012061688675882507, 0.013674186945130246], "top": "supported", "option_logits": null, "prompt_sha256": "60a0e925e3dcadbeab6b6a2b019538d0849b0ad4ff292f4f4dfabd004fc26fbe", "input_tokens": 209, "server_ms": 35.27}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.26291102776304, "probs": [0.025549816903577924, 0.9129549413995451, 0.06149524169687701], "top": "insufficient", "option_logits": null, "prompt_sha256": "aa7bc2d08112a9522d81345ac9a7d53f0d585926b12a5796bbf5cf6ed40654a7", "input_tokens": 207, "server_ms": 35.57}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.457533988170326, "probs": [0.07949528199138332, 0.8624138458567759, 0.05809087215184071], "top": "prohibited", "option_logits": null, "prompt_sha256": "d360c81abdb832d6d4fc247f34526c0acf5c10bf63f3ba2f0a5df79a0a0b84fe", "input_tokens": 228, "server_ms": 36.07}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.07228500209749, "probs": [0.2496799263499702, 0.5300831040455648, 0.220236969604465], "top": "prohibited", "option_logits": null, "prompt_sha256": "765f101f81708bceab877ada604f7eb0109387886d2c71372ad54f27f41596b7", "input_tokens": 229, "server_ms": 35.79}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.967536019627005, "probs": [0.05327692281408309, 0.050037135750001537, 0.8966859414359155], "top": "permitted", "option_logits": null, "prompt_sha256": "7c022953e71f15c5db0804d15f429edc65df946b8118edb4bed287be99eb8e06", "input_tokens": 222, "server_ms": 35.61}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.26408495986834, "probs": [0.9496834930411857, 0.02201796781359454, 0.0282985391452198], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a20ce272c75f01a4392c5d1b820d39545737e02ae5d1532f5088b556aaf2dfb", "input_tokens": 221, "server_ms": 35.94}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.04976097634062, "probs": [0.15783373213940913, 0.13075543386622587, 0.7114108339943651], "top": "A", "option_logits": null, "prompt_sha256": "20950a74ee9e9289ced5b4ab39da6c82ddcf60751467e3833c7b6268a969231a", "input_tokens": 216, "server_ms": 35.3}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.13483903976157, "probs": [0.04402789073014046, 0.9523922982664232, 0.003579811003436374], "top": "A", "option_logits": null, "prompt_sha256": "0c5c6af7ff5ee4ea09f5e5b6b8a9bb61a17684c1006f4cb8a05bae580e850500", "input_tokens": 210, "server_ms": 35.68}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.15850999997929, "probs": [0.057300813469286845, 0.036934646170185034, 0.905764540360528], "top": "A", "option_logits": null, "prompt_sha256": "64d5868d09c010e1d2b657f2aa408443a740902dc4db9140297cdb9bf9c68726", "input_tokens": 209, "server_ms": 35.28}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.54648798331618, "probs": [0.11336351747405907, 0.09391462502504651, 0.7927218575008944], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f73cc713484c1229bcbe4b2ff3834fd1d956866e23f823ba3db55ceff1aab63", "input_tokens": 211, "server_ms": 35.71}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 60.750422009732574, "probs": [0.9627223539065123, 0.013512193531635124, 0.023765452561852595], "top": "contradicted", "option_logits": null, "prompt_sha256": "18d2e56a6f70a329938566cb8f600ff8300b8df476d53d421cc99e8f0fac865d", "input_tokens": 226, "server_ms": 35.82}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.437106009572744, "probs": [0.1334863221372416, 0.0431515389781823, 0.8233621388845761], "top": "supported", "option_logits": null, "prompt_sha256": "4c97dcf4ec740144ccca8f21864fcded1b23ca85164465873e863616d563bd04", "input_tokens": 227, "server_ms": 35.88}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.6634010495618, "probs": [0.9756839658580125, 0.007785987664731666, 0.016530046477255724], "top": "supported", "option_logits": null, "prompt_sha256": "4f19b930e60acfcc87e3ba2e159c3b6e838e2d2684d3e150886db057cca7d973", "input_tokens": 202, "server_ms": 35.23}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.737008944153786, "probs": [0.961421480275785, 0.022290108712649817, 0.016288411011565182], "top": "insufficient", "option_logits": null, "prompt_sha256": "4fa649e0cb3d3f1aa6b1462f5c9aea273cd2e79799099f3857121c1ed5f99cc6", "input_tokens": 202, "server_ms": 35.51}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.671201003249735, "probs": [0.729412594713007, 0.22145529290309635, 0.049132112383896635], "top": "prohibited", "option_logits": null, "prompt_sha256": "26bab341e783c473bb0cd15ee4006b6de195a4ace87600b8f5d9a6057783c229", "input_tokens": 233, "server_ms": 35.29}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.01573498966172, "probs": [0.5597026262056906, 0.280700836596039, 0.15959653719827038], "top": "permitted", "option_logits": null, "prompt_sha256": "cc6d15a873dadb0a72b38963087b2306793d285516de46d7f2fe206334dfb815", "input_tokens": 231, "server_ms": 35.38}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.509764024056494, "probs": [0.06078538919398344, 0.03679812129727697, 0.9024164895087395], "top": "permitted", "option_logits": null, "prompt_sha256": "43ff1e21f8ff1c12d93e490c7bb0d0387a834684b19671df21468b5df5b419a5", "input_tokens": 216, "server_ms": 34.93}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.05953899025917, "probs": [0.3410513416651591, 0.6386849687911409, 0.020263689543699993], "top": "insufficient", "option_logits": null, "prompt_sha256": "963fcf1c25348b461b0716c138376d0bb6e4019f9668707e3835e3e78847a10d", "input_tokens": 223, "server_ms": 35.57}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.88559802668169, "probs": [0.0703544524474239, 0.9213105784571893, 0.008334969095386698], "top": "A", "option_logits": null, "prompt_sha256": "551ce45507e2c14daa97c398aa5de9a0b8cfe76a767653aa71c04f7b3abd5e3d", "input_tokens": 222, "server_ms": 35.33}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.449010010808706, "probs": [0.025341481023768493, 0.06914791484815831, 0.9055106041280732], "top": "B", "option_logits": null, "prompt_sha256": "2c7cebc2af92770f2b34548869b76e811f2a57f4d9411aa44574245c4be08d50", "input_tokens": 222, "server_ms": 35.37}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.16023004706949, "probs": [0.036501501263329385, 0.06835615589081268, 0.8951423428458579], "top": "B", "option_logits": null, "prompt_sha256": "2308c8cfd1b4ffa99ae77bf545c7e2bd04a3ab8dc7f03c3f731311c52ca45ca6", "input_tokens": 215, "server_ms": 35.05}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.05614199116826, "probs": [0.04743672482010346, 0.04743672482010346, 0.905126550359793], "top": "insufficient", "option_logits": null, "prompt_sha256": "ad22af27f0ad7843a439e48e5d02123c5db59fd008f5f14b01dc656541c67653", "input_tokens": 201, "server_ms": 35.2}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.10527397179976, "probs": [0.9152149580032972, 0.039736390129000654, 0.04504865186770221], "top": "contradicted", "option_logits": null, "prompt_sha256": "8c0be41435b431d759d31faa321275876446dcf2914fc5e6764726770b3c1db3", "input_tokens": 223, "server_ms": 35.32}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.62585901841521, "probs": [0.7192045412466151, 0.033247914805510814, 0.24754754394787407], "top": "supported", "option_logits": null, "prompt_sha256": "0815fedf6251d98db834d0623439ebb260d1a4f3bbfb6b661e7b1698ce47bd38", "input_tokens": 219, "server_ms": 35.35}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.28894597571343, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "32276207f151f7512091a9a8eeba70b47c1e41f955682b904a9db34d1ecf62c5", "input_tokens": 220, "server_ms": 35.32}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.16772300284356, "probs": [0.8363426091871619, 0.12734544467957343, 0.03631194613326466], "top": "insufficient", "option_logits": null, "prompt_sha256": "a7a409ea5b81586a7a32b1eb278d5201016f321cc2071a106973bfccb58d081c", "input_tokens": 218, "server_ms": 35.34}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.12578902905807, "probs": [0.07444288610891006, 0.06566437538338366, 0.8598927385077062], "top": "prohibited", "option_logits": null, "prompt_sha256": "62f8bfce50f960f55352dd75b5cdcc21b872cdf92f9e0a0468dfd37b0286b310", "input_tokens": 224, "server_ms": 35.47}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.82443300914019, "probs": [0.33994849673548966, 0.13265166636914294, 0.5273998368953674], "top": "prohibited", "option_logits": null, "prompt_sha256": "9cf8ec9573bb1975e6a04867c89de97b537f00b37ac0b00bfc2d94409177f946", "input_tokens": 221, "server_ms": 35.48}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.6651949621737, "probs": [0.8736778955723182, 0.0805335711500283, 0.04578853327765339], "top": "permitted", "option_logits": null, "prompt_sha256": "c6d0dce9defd2edc46e21f6bf2e2ea73cdb9a2f27c00e61e4a29ea8091c80c7a", "input_tokens": 221, "server_ms": 35.13}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.90125699341297, "probs": [0.062121585338401536, 0.9222535863628694, 0.015624828298729139], "top": "insufficient", "option_logits": null, "prompt_sha256": "cdbf0d0be3a9e0d55a656ba7af8a3f8810e109f3a4412511847270b44e9ab030", "input_tokens": 220, "server_ms": 35.44}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.41150698950514, "probs": [0.0306838713939516, 0.9671154885765151, 0.0022006400295334033], "top": "A", "option_logits": null, "prompt_sha256": "5524c9598304299c0fae65bf8bf8ad44801cdcb1bd308012e90f5ded20b6ba67", "input_tokens": 227, "server_ms": 35.1}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.82431799080223, "probs": [0.018707424584872102, 0.9739927314924995, 0.007299843922628353], "top": "B", "option_logits": null, "prompt_sha256": "046ec0b3a43e34bd3ad9db12de82d9ab13469716689b9965c7d95a66c352e39c", "input_tokens": 230, "server_ms": 35.53}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.97968699596822, "probs": [0.018604370570790937, 0.9686272786061052, 0.012768350823103862], "top": "B", "option_logits": null, "prompt_sha256": "90d9b06fa0f1feef4f1dae43f365de42ae5f316337ec9a7aa0aa6aa37c669a98", "input_tokens": 223, "server_ms": 35.07}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 51.80186202051118, "probs": [0.02713643902676001, 0.9696486773304461, 0.003214883642793822], "top": "insufficient", "option_logits": null, "prompt_sha256": "85c14e214fe05ffb55d500ad4af7eb7345fc6dc8977d072368298d19b856706f", "input_tokens": 210, "server_ms": 35.12}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.4060630123131, "probs": [0.8801051546539677, 0.027923276514473683, 0.09197156883155871], "top": "contradicted", "option_logits": null, "prompt_sha256": "47f18c1ebc5b12492bc779a1680bd2655ef7a5e1c51231980a4f2044a1fd2c16", "input_tokens": 227, "server_ms": 35.13}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.29296999843791, "probs": [0.08978367791396158, 0.051047811831364455, 0.859168510254674], "top": "supported", "option_logits": null, "prompt_sha256": "e167d21bdf4613da98264568b0f7e46b1dc7e5d439bf20bb1e194ad8a1faa97c", "input_tokens": 224, "server_ms": 35.18}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.90840601082891, "probs": [0.5781390325410648, 0.3087204270019537, 0.1131405404569815], "top": "contradicted", "option_logits": null, "prompt_sha256": "09d93074ccda5bea0013692c06402396cf588d13622e29fa281b715aa32bd8e1", "input_tokens": 220, "server_ms": 34.99}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.98445500759408, "probs": [0.055608297494421005, 0.008467179367297219, 0.9359245231382817], "top": "insufficient", "option_logits": null, "prompt_sha256": "237d0434d6c9553e9da609de77e16fe67eab4b3dd0651270af00d8bd1d002fb4", "input_tokens": 208, "server_ms": 35.46}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.50880097830668, "probs": [0.12883641826967107, 0.2128198779442895, 0.6583437037860393], "top": "permitted", "option_logits": null, "prompt_sha256": "847503511d164b28bae40eb9a26c124b4f5604c93c23c64e73ab4c0f4a8b7674", "input_tokens": 227, "server_ms": 35.13}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.05598704237491, "probs": [0.3671636825410607, 0.02633289140285521, 0.6065034260560841], "top": "permitted", "option_logits": null, "prompt_sha256": "fe9be4c9a4653da33139e911ffc27851336005ffef91ea116405e4867a22df2c", "input_tokens": 228, "server_ms": 35.3}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.937245964538306, "probs": [0.886900606305962, 0.06360830231476525, 0.04949109137927273], "top": "permitted", "option_logits": null, "prompt_sha256": "f047d6c0b9b2da3679e9f8e44e33924d72e04dee96588064644813ed8fba2e8a", "input_tokens": 225, "server_ms": 35.22}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.0822820039466, "probs": [0.0734095390557655, 0.02373077921642233, 0.9028596817278121], "top": "insufficient", "option_logits": null, "prompt_sha256": "61743187961a10bc606b2e07c7f3e07006de441253cacdf0ad34c0d93de10206", "input_tokens": 218, "server_ms": 35.23}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.89278300572187, "probs": [0.6690253227621226, 0.06991358622745607, 0.26106109101042135], "top": "A", "option_logits": null, "prompt_sha256": "4b7526d0b4eeb5242715b4607249dc7fda6cf97c71dcc902cd046f73925448e7", "input_tokens": 221, "server_ms": 34.87}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.468425991013646, "probs": [0.16680897363694863, 0.8005468752842061, 0.03264415107884514], "top": "A", "option_logits": null, "prompt_sha256": "4408aba1facc73fb41856f6e1dc6c5b0d101515eaa9d3d36c998d455ca98f352", "input_tokens": 227, "server_ms": 35.26}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.00174199510366, "probs": [0.6472762711048763, 0.324620580367349, 0.02810314852777479], "top": "A", "option_logits": null, "prompt_sha256": "0b110642e6f85b6b9af06ece23e13205ce08a034a618743f303b28e850df3715", "input_tokens": 207, "server_ms": 34.96}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.42636501882225, "probs": [0.9579483616410114, 0.015242642754936796, 0.026808995604051852], "top": "insufficient", "option_logits": null, "prompt_sha256": "45b0316cd9c277a9bb15a11f91bd859ef40e49a3ab00f161e1eb488c6e0e7b0c", "input_tokens": 197, "server_ms": 35.31}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.94799600960687, "probs": [0.9718556905482565, 0.017531269220884715, 0.010613040230858772], "top": "contradicted", "option_logits": null, "prompt_sha256": "cdb3c0f15d25126f3098fd8687670ff98df334e531fe734a90fd20964e3c9f10", "input_tokens": 213, "server_ms": 35.07}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.70999197475612, "probs": [0.019733010925224156, 0.015353470531745737, 0.9649135185430301], "top": "supported", "option_logits": null, "prompt_sha256": "b4e0e9b59a46be9303b34ef8102e1d9111539bea553263938df2bfc22c9dd274", "input_tokens": 212, "server_ms": 35.4}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.98078704113141, "probs": [0.9709235969179878, 0.013627301203680551, 0.015449101878331626], "top": "supported", "option_logits": null, "prompt_sha256": "e5eaec4c174f6c30c9c81367eae7d66e2ba382d18bb05436473d679cafb24334", "input_tokens": 202, "server_ms": 35.07}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.060571012087166, "probs": [0.8976195797221951, 0.01428270535936492, 0.08809771491844004], "top": "contradicted", "option_logits": null, "prompt_sha256": "df64bd34d0d81d7b47be5dbc60eef67a495ee5f90e2504e9cf782b6a9d05686e", "input_tokens": 203, "server_ms": 35.53}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.96802502218634, "probs": [0.8705748790556944, 0.058640587741848625, 0.07078453320245706], "top": "prohibited", "option_logits": null, "prompt_sha256": "fd7e983dea8e27e2db4d4476f707a33c8f1988072e046e59b8dde274d605403a", "input_tokens": 218, "server_ms": 35.27}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.32764102099463, "probs": [0.03565003212414025, 0.9308678254479662, 0.03348214242789346], "top": "permitted", "option_logits": null, "prompt_sha256": "1cb99992bb26c8f1538437106767655293f477f4946f1a084a3f1fc536d5db41", "input_tokens": 221, "server_ms": 35.6}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.477289038710296, "probs": [0.9356609917402224, 0.026185265088641236, 0.038153743171136366], "top": "permitted", "option_logits": null, "prompt_sha256": "a120de7fa181e55bda051aec589d2be6dff9056b71f321879d28e3c6f0524a48", "input_tokens": 215, "server_ms": 35.1}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.607102017849684, "probs": [0.17108244800120995, 0.2654191980997831, 0.5634983538990069], "top": "prohibited", "option_logits": null, "prompt_sha256": "3f00e519658a47dabd139558a1a8c9c19ec33a397c79d6f38432fcd92ffa6759", "input_tokens": 220, "server_ms": 35.31}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.65925801359117, "probs": [0.02654524013373213, 0.02493101477406087, 0.948523745092207], "top": "A", "option_logits": null, "prompt_sha256": "75e6aa62212ed77b1b8e3372e11c267ce5041413067bf12b523300be3f1d2c8a", "input_tokens": 206, "server_ms": 35.32}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.57338901376352, "probs": [0.9767856508491533, 0.019975802811812802, 0.003238546339033884], "top": "B", "option_logits": null, "prompt_sha256": "a4086e6641a8cfb71d89c50e81af6a0d26c78a0e2cca3b111a59d07fc5534227", "input_tokens": 212, "server_ms": 35.28}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.04649699246511, "probs": [0.9722335117661297, 0.025554202312284887, 0.002212285921585334], "top": "B", "option_logits": null, "prompt_sha256": "b4d0de3cb9bc5006b85a9ab8defc48bc023a8160179f1ac5ef0afc35737db2fc", "input_tokens": 203, "server_ms": 35.2}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.946144005749375, "probs": [0.04360114802649768, 0.943161207167596, 0.013237644805906336], "top": "insufficient", "option_logits": null, "prompt_sha256": "71c48e87b6408ec5edeb911ec9c333e5314ca1902ea38d67e60b89e3733ec662", "input_tokens": 191, "server_ms": 35.3}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.293913952074945, "probs": [0.08579102787436829, 0.2340933015047472, 0.6801156706208845], "top": "contradicted", "option_logits": null, "prompt_sha256": "57826b5b8530ae304d16f591c4dd35a0e6c6c3d66d0015b8656598b752d6b56d", "input_tokens": 224, "server_ms": 35.11}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.169035993050784, "probs": [0.06220125487024129, 0.8672819548613933, 0.07051679026836535], "top": "supported", "option_logits": null, "prompt_sha256": "8fccfa0887a064fc1c7381405d0093e3f490b0d8062590c34349334bf93a1a91", "input_tokens": 221, "server_ms": 35.09}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.45642298925668, "probs": [0.04931839214968541, 0.009651501126706794, 0.9410301067236078], "top": "supported", "option_logits": null, "prompt_sha256": "b8d01c5f1cfc65c56440853bb6ea71da814dc8d2dbdbe0b6ebe49bef67f4a9bc", "input_tokens": 229, "server_ms": 35.03}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.99895895970985, "probs": [0.9702339069518919, 0.02249442043151019, 0.007271672616597835], "top": "insufficient", "option_logits": null, "prompt_sha256": "1229efd0e44b4ec6cd2d562289154ff5c11c31e86264a7ce7e635b98cc697086", "input_tokens": 226, "server_ms": 35.55}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.32676798570901, "probs": [0.2513704799344376, 0.6441910381357905, 0.10443848192977191], "top": "permitted", "option_logits": null, "prompt_sha256": "6c3289b3c60badefade24875b521714e75aa53b0674872edb9f0708213f84ad8", "input_tokens": 226, "server_ms": 35.1}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.92403801670298, "probs": [0.031035471572559672, 0.8103757740468936, 0.1585887543805467], "top": "permitted", "option_logits": null, "prompt_sha256": "05edfab6235151954638b9a2956274333d03a9fdd4c7cec72b3136cd8497d055", "input_tokens": 215, "server_ms": 35.37}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.644413012079895, "probs": [0.28260348014973824, 0.5999814666465165, 0.11741505320374515], "top": "permitted", "option_logits": null, "prompt_sha256": "f2ed20d5d37c47c360b87dd6fabb7aa18e2a92bf8e7924bbcd02f4d37f76b501", "input_tokens": 225, "server_ms": 35.1}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.61198802711442, "probs": [0.6006752256036975, 0.0356893377380391, 0.3636354366582635], "top": "insufficient", "option_logits": null, "prompt_sha256": "81ac90c620f667eaca4acad99002b68effb931cfae4551f8bf7306013acb86ee", "input_tokens": 224, "server_ms": 35.42}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.53694799123332, "probs": [0.16176938835030455, 0.8266285492101126, 0.011602062439582941], "top": "A", "option_logits": null, "prompt_sha256": "35ad7c6e27133453e50f181f7eddddc3842f0f2da4c316fc7ce85255f43468a2", "input_tokens": 213, "server_ms": 35.01}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.702106000855565, "probs": [0.182989315672784, 0.6832919497683837, 0.1337187345588323], "top": "B", "option_logits": null, "prompt_sha256": "cc9b1512a6abeeaacf0aba92a0b32b5d718db9c45bb9f31928cb8d95ceb1e152", "input_tokens": 218, "server_ms": 35.28}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.21302398806438, "probs": [0.027898126913861773, 0.03585599816809988, 0.9362458749180382], "top": "B", "option_logits": null, "prompt_sha256": "26c7541156fe8ba354870323e878002e46ad331c7ffa7782fb6df1051840ced3", "input_tokens": 209, "server_ms": 34.87}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.97124501178041, "probs": [0.919161537118523, 0.022690124115183632, 0.05814833876629327], "top": "insufficient", "option_logits": null, "prompt_sha256": "904cfa01a76941d34f2964642a3841ba68802c89aaa878cc4dbbdbefc0084ec0", "input_tokens": 207, "server_ms": 35.5}, {"id": "14790d6d50043b03420c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.752326988615096, "probs": [0.9594367541920572, 0.032411181205817395, 0.008152064602125513], "top": "contradicted", "option_logits": null, "prompt_sha256": "6742d19e6f2f7e8cc85c08be7b038e5ceeabb7b63c5bcdc82e120416a0bce676", "input_tokens": 204, "server_ms": 35.11}, {"id": "29566b5efbd2611e6eee", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.36492497008294, "probs": [0.041513347298601795, 0.06048782007179681, 0.8979988326296015], "top": "contradicted", "option_logits": null, "prompt_sha256": "d6dab4bfc6107fe33afbdd7a53a851050d3498e882efca1ed4679c8fee89b60c", "input_tokens": 215, "server_ms": 35.13}, {"id": "1a6e02f5b0bf923374c1", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.48218698054552, "probs": [0.035722764629231414, 0.03151023805566489, 0.9327669973151036], "top": "contradicted", "option_logits": null, "prompt_sha256": "1275c500e69ee4e3c779b72b546c1079fd5259f9b00b0ea2a9727ea78ae42f12", "input_tokens": 236, "server_ms": 35.51}, {"id": "d83888299b2e202297be", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.689247007947415, "probs": [0.02669293589117738, 0.01950575982260376, 0.9538013042862189], "top": "supported", "option_logits": null, "prompt_sha256": "79dad3515d6ac33938250c57cbc27d196b71bc3e8c7c46557f6ffcc440599486", "input_tokens": 209, "server_ms": 35.06}, {"id": "f42c36cd140557859819", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.58558999048546, "probs": [0.9575581678036222, 0.017273357092855306, 0.025168475103522527], "top": "supported", "option_logits": null, "prompt_sha256": "6c82712d126b30080db5961980e2188bd98cbe7185a80b9d0850476582611255", "input_tokens": 220, "server_ms": 34.89}, {"id": "c2e8c6d1252eabe3e182", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.56593197211623, "probs": [0.9756491443197615, 0.008826562635325764, 0.015524293044912726], "top": "supported", "option_logits": null, "prompt_sha256": "9f99e79be8bfc744e8e881df8765962b2c5e9ead7c08aa3031d8c4e482b94314", "input_tokens": 241, "server_ms": 35.47}, {"id": "70d7bdc8d5ee62366ee5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.01322299055755, "probs": [0.9671640812547988, 0.01744663776234453, 0.015389280982856728], "top": "contradicted", "option_logits": null, "prompt_sha256": "1cd67b493a05a62ee45b1f9e1d6661a965f7c67c1abdff0f2f900efab7b44c57", "input_tokens": 194, "server_ms": 34.94}, {"id": "26d6f758382576f65927", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.534495003987104, "probs": [0.03395990035306926, 0.021889687043195855, 0.9441504126037349], "top": "contradicted", "option_logits": null, "prompt_sha256": "c7c2ec7e5bea535be8354d70b4f4d45ee27f85b2d709c37228631d2256ffa787", "input_tokens": 205, "server_ms": 35.21}, {"id": "ecc5ee8175d414efdd35", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.5030189785175, "probs": [0.0320815187583301, 0.018240413999121397, 0.9496780672425486], "top": "contradicted", "option_logits": null, "prompt_sha256": "8cbd6e164e8602679ed55521487a9916e142fd6b0db9f61afed7f12f33694041", "input_tokens": 226, "server_ms": 35.07}, {"id": "21ecfcb1d048958cb9b4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.58512798557058, "probs": [0.9732459058279692, 0.019903413035453177, 0.006850681136577472], "top": "supported", "option_logits": null, "prompt_sha256": "6383fc2f4c47375763c47b0be34a9e5cdf31d37feb029b9be2cf2d5a15e0772e", "input_tokens": 205, "server_ms": 35.07}, {"id": "5ce4f874ff2fa5295685", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.68098698090762, "probs": [0.01962200546026409, 0.02089248498400619, 0.9594855095557296], "top": "supported", "option_logits": null, "prompt_sha256": "f89d86e354b4d4a024f8ff4da9a22724b4cbde6291d1a5b0f449ca51c6767462", "input_tokens": 216, "server_ms": 35.07}, {"id": "04d55440accab2c5d0d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5/stability", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.93430596450344, "probs": [0.01358384149332632, 0.018589002082326323, 0.9678271564243475], "top": "supported", "option_logits": null, "prompt_sha256": "ef4e6a248e0f070f79f76c81c90b0f539bb69b4a899af662f52f13cc99835adb", "input_tokens": 237, "server_ms": 35.41}, {"id": "0e2caab29b17ff00a2c8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.68318600812927, "probs": [0.05252720299692694, 0.0634051241850924, 0.8840676728179807], "top": "contradicted", "option_logits": null, "prompt_sha256": "ddd0c1b1e52b902b4601f11c4e01cdfc4b5d13191503568ad938eb43627f65de", "input_tokens": 198, "server_ms": 35.3}, {"id": "a4428ee66c63c4f1e7f4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.546700975857675, "probs": [0.911447144098782, 0.061393677970427434, 0.02715917793079057], "top": "contradicted", "option_logits": null, "prompt_sha256": "391ac6a08068524e71fbebee117ce4b465409471026beed52d46e9d5a31c738d", "input_tokens": 209, "server_ms": 35.19}, {"id": "22e54b5c44f60d5cd5c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.480957983992994, "probs": [0.9575581678036222, 0.025168475103522527, 0.017273357092855306], "top": "contradicted", "option_logits": null, "prompt_sha256": "4462342da6b4d2c906a689690d3327c87295e9e6d8acf59f67720f93f62f7de6", "input_tokens": 230, "server_ms": 35.35}, {"id": "9729054768eb7f90eb6d", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.40585403610021, "probs": [0.8908371534836812, 0.016069778438424883, 0.09309306807789393], "top": "supported", "option_logits": null, "prompt_sha256": "d31e029e326da28a1afbe7077530e031d890aadb023515156766e33e7e6858df", "input_tokens": 211, "server_ms": 35.28}, {"id": "674217922d611e718129", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.289143008645624, "probs": [0.040946062905879925, 0.01597760756604808, 0.943076329528072], "top": "supported", "option_logits": null, "prompt_sha256": "71a2d31f4be12a2b914ca68f9cae390500f93822e3cb3874d7f64f5c64fe6763", "input_tokens": 222, "server_ms": 35.31}, {"id": "7a8a295f17d1296482dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c/stability", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.007422004360706, "probs": [0.030189528318779257, 0.018276068996626402, 0.9515344026845942], "top": "supported", "option_logits": null, "prompt_sha256": "78fad6e4cc471906238d1886c555f8250939c7436ef979277a07dc1973459f64", "input_tokens": 243, "server_ms": 35.28}, {"id": "8bceb31cf793fb26d1a5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.63118398236111, "probs": [0.12378842710422985, 0.7171128049687485, 0.15909876792702154], "top": "prohibited", "option_logits": null, "prompt_sha256": "13a5c4f6441c3dd3120f57edec9643adbbecf11e7645ae62e42f4c2ff66326c2", "input_tokens": 212, "server_ms": 35.12}, {"id": "2e895a0fc82d936936aa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.90643997816369, "probs": [0.12505351454311125, 0.7713474203984862, 0.10359906505840241], "top": "prohibited", "option_logits": null, "prompt_sha256": "ce49dcac69aa2af6e12b1c17b2a43f21aaa0127be15c7318829e2642b2bd7c2f", "input_tokens": 223, "server_ms": 35.17}, {"id": "c5a9fbb9bfa41e5293eb", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.63525199936703, "probs": [0.08849507711341235, 0.8468374918735759, 0.06466743101301171], "top": "prohibited", "option_logits": null, "prompt_sha256": "fdd085f4ab84e3a018ecbe392c6a2b44241a67617e4f6922d3ee43d4c7aafe69", "input_tokens": 244, "server_ms": 35.27}, {"id": "80970cf7e755a58a049e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.95186896203086, "probs": [0.04039650215413246, 0.08576381590487972, 0.8738396819409878], "top": "permitted", "option_logits": null, "prompt_sha256": "9258fbb9a33f8d5daca32c9cc0f5a10a53e97b8a083863a22103ee6834a3536e", "input_tokens": 209, "server_ms": 35.36}, {"id": "d300f689be3cf2daa666", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.412346984259784, "probs": [0.7210154897161607, 0.26423934747358174, 0.014745162810257588], "top": "permitted", "option_logits": null, "prompt_sha256": "4cabde88bd2a21ea894b4062f3ecd8acee3e430adaf4c391717ce65393a8eda6", "input_tokens": 220, "server_ms": 35.01}, {"id": "3d8f3be170e01d2c78e5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.73725004028529, "probs": [0.9196237689874805, 0.05463978191057977, 0.025736449101939722], "top": "permitted", "option_logits": null, "prompt_sha256": "9e9b7d32c9ac3fb0e95c28a7ea6c02500784731ef666d2a29e72ed45ac08d6aa", "input_tokens": 241, "server_ms": 35.3}, {"id": "aef33dfba4338b1af990", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.906209020875394, "probs": [0.9040272786001343, 0.06903464008509737, 0.026938081314768388], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ec83bd288943dc7a99a59417d5d4c84de7a99c12f6c7d10dedbfa6f23c1618f", "input_tokens": 208, "server_ms": 35.16}, {"id": "f806bba07a63e30f7181", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.28206095146015, "probs": [0.11317700613619622, 0.04416292269369592, 0.8426600711701079], "top": "prohibited", "option_logits": null, "prompt_sha256": "e4e6e4972cf7b8fda4ebaf016169e30457659386e4313065924a300d7ae92102", "input_tokens": 219, "server_ms": 35.47}, {"id": "409e1ab1d29db22f8917", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.05831599980593, "probs": [0.06149524169687701, 0.025549816903577924, 0.9129549413995451], "top": "prohibited", "option_logits": null, "prompt_sha256": "374c2786adbd61c0f48d5f6fa9e4c869e2a0ce62e10afc7f8995a321c3ac7b3e", "input_tokens": 240, "server_ms": 35.08}, {"id": "0471c62c50820b1d4203", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.38654899969697, "probs": [0.9562399673788796, 0.030338822060271976, 0.01342121056084844], "top": "permitted", "option_logits": null, "prompt_sha256": "12950b871ef0d808f0ffe5f627a6ab7fcff37d15b4056926bef82e5ca9515603", "input_tokens": 214, "server_ms": 35.34}, {"id": "10bac1f2a20209f3a4cd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.50992700550705, "probs": [0.038602800049854315, 0.07229125256770057, 0.8891059473824451], "top": "permitted", "option_logits": null, "prompt_sha256": "bb023006ec54375034ae4cf2429f5809e707e5b8130c82935dd54c329a753c81", "input_tokens": 225, "server_ms": 35.19}, {"id": "59898712e090848aa8b9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.99056402873248, "probs": [0.02897489209007431, 0.05777440406099018, 0.9132507038489355], "top": "permitted", "option_logits": null, "prompt_sha256": "da7448a956e2a207e4d60c49800356db2738858940a03e326e063f0bd32b231d", "input_tokens": 246, "server_ms": 35.58}, {"id": "20af0ba1b17131d525dd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.924940013326705, "probs": [0.05809087215184071, 0.07949528199138332, 0.8624138458567759], "top": "prohibited", "option_logits": null, "prompt_sha256": "1d6a69aaeb12d52a1ca3ad68f96cdffee869fdab2c4f5398747939797d63cc93", "input_tokens": 216, "server_ms": 34.95}, {"id": "c153f7547c7206472fda", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.96239600284025, "probs": [0.9248115837085638, 0.058505774506871465, 0.016682641784564755], "top": "prohibited", "option_logits": null, "prompt_sha256": "4b841741dc334a80d58728c6d17f3eeef9b1cc4ca29df8b4ad76fdca75d88438", "input_tokens": 227, "server_ms": 35.22}, {"id": "36d25b31f325cc4cec04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.812875994481146, "probs": [0.9558172950443494, 0.03437954200492852, 0.009803162950722118], "top": "prohibited", "option_logits": null, "prompt_sha256": "2202a032d3393bbdb9175be2a35b9df61926b5c5f05e2115ba421a2fc5689fb0", "input_tokens": 248, "server_ms": 35.22}, {"id": "d6a681055a01ed788b91", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.65297795692459, "probs": [0.3271724746782474, 0.6126940898961105, 0.06013343542564212], "top": "permitted", "option_logits": null, "prompt_sha256": "566d9a9e51a19ee5241f234f7fc4659b2b552124ee86b2a8abd5df8351291f59", "input_tokens": 209, "server_ms": 35.35}, {"id": "3cafd544b9a8b89091a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.62842004979029, "probs": [0.5382555047693012, 0.22363240950232563, 0.2381120857283732], "top": "prohibited", "option_logits": null, "prompt_sha256": "a9efefcbd3ee4c3b2c855d02e1bb037cff4591012c90d5f7bef9b62f4d4ccdcd", "input_tokens": 220, "server_ms": 34.95}, {"id": "24c4e34db7cfbd13c409", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027/stability", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.301609004847705, "probs": [0.09691612655904591, 0.7215895775500428, 0.1814942958909112], "top": "permitted", "option_logits": null, "prompt_sha256": "7ca1641e193289ea6cd31a7e5a35321da55413565d1467291ae3e493ca0f647d", "input_tokens": 241, "server_ms": 35.25}, {"id": "69a2a9d14f62016e74c7", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.23570198379457, "probs": [0.03055575430681714, 0.9630773803718248, 0.006366865321358165], "top": "A", "option_logits": null, "prompt_sha256": "e3a1608fbed3ced6202a44615ea2c5fb528aeef37542c6dba7be476e30bf35a0", "input_tokens": 200, "server_ms": 34.99}, {"id": "f0519e8daaa832f8c594", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.36758098565042, "probs": [0.04645865614250789, 0.9438608101873385, 0.009680533670153552], "top": "A", "option_logits": null, "prompt_sha256": "556b7e0ed68569295783d5fb39a74205d8de9d6a85b8b39753f2f2ab355a1316", "input_tokens": 211, "server_ms": 35.19}, {"id": "97b7222868f23ec1cdec", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.42615902191028, "probs": [0.041314150395761425, 0.9515541773156176, 0.007131672288621026], "top": "A", "option_logits": null, "prompt_sha256": "8ab5bcd8451d29e0a8b6a4e944e8ad55d5a69817e49f0253295b72368b6d97ab", "input_tokens": 232, "server_ms": 35.23}, {"id": "dfdd8717200917fdbee2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.97610500222072, "probs": [0.04937458794264477, 0.9421023427618396, 0.008523069295515533], "top": "B", "option_logits": null, "prompt_sha256": "c799fd6ddc093f048620d6153075c549e3fd78f75142ee29b4a549e6717cfb7a", "input_tokens": 208, "server_ms": 35.71}, {"id": "9bdf7dba759fc9919f7a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.25239698868245, "probs": [0.03170454578813462, 0.9385188723730931, 0.029776581838772298], "top": "B", "option_logits": null, "prompt_sha256": "919145894958c3132f3b9fad798c193374d28316f5d143abed763ee78bc6cb3c", "input_tokens": 219, "server_ms": 35.24}, {"id": "f32ffa0158733f44991c", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.50880301557481, "probs": [0.03837677130689544, 0.9411304198850982, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "55ce5ae50b004222181e000988464a63fbcc4d43d8c35dfc659291dfd23f7275", "input_tokens": 240, "server_ms": 35.24}, {"id": "e02cc1d78a9a02284d06", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.8727410021238, "probs": [0.07467635320167006, 0.9184401561944533, 0.006883490603876593], "top": "A", "option_logits": null, "prompt_sha256": "4fe3db2d4a51f2820d80430df3b9e19d8eb88e44035b349800a61b3be4b8d655", "input_tokens": 203, "server_ms": 35.44}, {"id": "870ee31666893f8a0298", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.075875039212406, "probs": [0.04934735545622599, 0.9415827314046187, 0.009069913139155384], "top": "A", "option_logits": null, "prompt_sha256": "17fcb1d337e2b8d379b62d646f04e35e95a1571c684b8005bbb8fe0bc75454e2", "input_tokens": 214, "server_ms": 35.15}, {"id": "ff15024839a7943a46f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.98976197466254, "probs": [0.1386663227728394, 0.8553131252619345, 0.006020551965226119], "top": "A", "option_logits": null, "prompt_sha256": "04e28f07ca59d6103bf8ea839eeb4acad9d5f22bbc97fbd246055b2397d46eeb", "input_tokens": 235, "server_ms": 35.35}, {"id": "d73c99ef45d5d49688c5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.14993904903531, "probs": [0.028752813873309538, 0.0649961125421044, 0.9062510735845861], "top": "B", "option_logits": null, "prompt_sha256": "17dd4100e3a564160d483a95aa933134109f0af4b761a85d93752d4aff6354d2", "input_tokens": 203, "server_ms": 35.21}, {"id": "10777cd59d67780f2938", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.48596000112593, "probs": [0.9613258651902599, 0.03457767929205241, 0.004096455517687579], "top": "B", "option_logits": null, "prompt_sha256": "a1e3c3c44e04e96a93d4ce337899568052893a7fec883864ba50a7a29b34ce00", "input_tokens": 214, "server_ms": 35.24}, {"id": "1e1feb87b303b0c66ca6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.089794993866235, "probs": [0.9690035403166097, 0.027118383844547535, 0.003878075838842631], "top": "B", "option_logits": null, "prompt_sha256": "fd3a20c512e095fff4a5d2d1a3292b87f405a644dbe8b65df50913e10909f177", "input_tokens": 235, "server_ms": 35.4}, {"id": "c62600144038b1f20132", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.709478011820465, "probs": [0.04668393422657173, 0.9484375641457526, 0.004878501627675742], "top": "A", "option_logits": null, "prompt_sha256": "784f9063eff07c0cb2a7523a0e0d86f85f0e30e28b315ad6fdb900623c46dae3", "input_tokens": 208, "server_ms": 35.0}, {"id": "0c34b0762c1f92a806fa", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.989320014603436, "probs": [0.041364760682874986, 0.9527198548965369, 0.005915384420588236], "top": "A", "option_logits": null, "prompt_sha256": "4148610583b5757c6250bdfe517dd5242b26bc51b84860cc0477920853ee96b5", "input_tokens": 219, "server_ms": 35.19}, {"id": "d5f4985849895549f014", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.04717301623896, "probs": [0.03455875256351939, 0.9607996887353764, 0.004641558701104165], "top": "A", "option_logits": null, "prompt_sha256": "e065ccba9e82679941e016e4d903fefbb0b0541ab6c289424d8406646b8a4df4", "input_tokens": 240, "server_ms": 35.03}, {"id": "9ad92b012bc3b7b1298b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.52366901934147, "probs": [0.03260024638770391, 0.06105030489607428, 0.9063494487162218], "top": "B", "option_logits": null, "prompt_sha256": "6a670a653ed4416de565366b725bb779a75ff88735fb24c72f7618515b927dd0", "input_tokens": 206, "server_ms": 35.09}, {"id": "9bfefd9320c1ec9dc05a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.872485015541315, "probs": [0.9673050958541622, 0.028823625851960685, 0.003871278293877062], "top": "B", "option_logits": null, "prompt_sha256": "be4659bbe8df7f37810835bed560d81ef0117f836740d908d5ef70fff1ff6691", "input_tokens": 217, "server_ms": 35.18}, {"id": "7062c93ddaabe9b19133", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.48280497128144, "probs": [0.9747279202332917, 0.019933720411485507, 0.005338359355222771], "top": "B", "option_logits": null, "prompt_sha256": "b1d3c293c0e4f7d3ce55e8c99b2bd1d8e76ac080f813a977ba1cd8bdfa24c6bb", "input_tokens": 238, "server_ms": 35.32}, {"id": "a6861c422c6374527b79", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.90299798455089, "probs": [0.024123816409904218, 0.05806303500527026, 0.9178131485848255], "top": "contradicted", "option_logits": null, "prompt_sha256": "3e2617b0ad974977c0dd1594df406bb3034f6fcb933ce69d16ad8d4b09064f24", "input_tokens": 216, "server_ms": 35.01}, {"id": "868cc04ff6c8321975c2", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.86704398551956, "probs": [0.9398897417986248, 0.026303609545220985, 0.0338066486561543], "top": "contradicted", "option_logits": null, "prompt_sha256": "2a3b7f86609a9053a5df8ff36e0da214bccb9836625b9a9f61b5de686789c6ff", "input_tokens": 227, "server_ms": 35.38}, {"id": "5763d6ac9583bfbd179a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.509348013903946, "probs": [0.9594855095557296, 0.01962200546026409, 0.02089248498400619], "top": "contradicted", "option_logits": null, "prompt_sha256": "6b6e87feb0f69a462bbac03b3a26eb92b9b8894940c1961d86676868379ea042", "input_tokens": 248, "server_ms": 34.91}, {"id": "7c1aefb79903e6234b4b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.47160203382373, "probs": [0.10854545315280983, 0.8605033267713885, 0.03095122007580161], "top": "prohibited", "option_logits": null, "prompt_sha256": "45fd8e17d252c809a79b82f9efd9703cf2ed20d705ef4e9f6a8263632518c905", "input_tokens": 228, "server_ms": 35.24}, {"id": "bd04047294bf97c56669", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.55960701126605, "probs": [0.08812608673880612, 0.8433065123136125, 0.06856740094758139], "top": "prohibited", "option_logits": null, "prompt_sha256": "1afe00165f080968c33c908744c05088da020a9e82218b97d806830f43589e42", "input_tokens": 239, "server_ms": 35.13}, {"id": "c5e09f0caa6939ae1c04", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.538435022346675, "probs": [0.05131106478236839, 0.9195152853860296, 0.02917364983160216], "top": "prohibited", "option_logits": null, "prompt_sha256": "031d737c37240c7d584079eec46f32841816b67fac15c3335fb2793ea98a7f39", "input_tokens": 260, "server_ms": 35.68}, {"id": "54e50afc4284f5e45e99", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.13569400459528, "probs": [0.8285128513585246, 0.15227846530972178, 0.019208683331753663], "top": "A", "option_logits": null, "prompt_sha256": "31ec20ea46211c6be051b3eaf26eebf58489c47d676c5ba4099bb86e2579bd80", "input_tokens": 216, "server_ms": 34.85}, {"id": "84b0fc54c146eacf4b63", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.54725901270285, "probs": [0.13323523151808067, 0.1418619009264831, 0.7249028675554362], "top": "A", "option_logits": null, "prompt_sha256": "27a249ed911cf472fa247ebba26c0c83c6918291bdec9ca8691b369c0ed2aa56", "input_tokens": 227, "server_ms": 35.3}, {"id": "0b580003cbe6b93cefd6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe/stability", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.13877498358488, "probs": [0.11080847491729577, 0.1614556091001449, 0.7277359159825594], "top": "A", "option_logits": null, "prompt_sha256": "c34bcb827dc858e98dafcc3d569a92e6c59e6dbb3c4f9555c63cf1da8fe0f387", "input_tokens": 248, "server_ms": 35.04}, {"id": "9e04a89f6da01a73345f", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.69186605094001, "probs": [0.034226185004474344, 0.014220169297068841, 0.9515536456984567], "top": "contradicted", "option_logits": null, "prompt_sha256": "dc93ce4ed40dfc816f75d2859160f06f6fdd717369cf8d59be8e0a7d9a52d734", "input_tokens": 226, "server_ms": 35.57}, {"id": "61052547b62b4f38f768", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.34740300895646, "probs": [0.9570038692718861, 0.016213567765142052, 0.026782562962971823], "top": "contradicted", "option_logits": null, "prompt_sha256": "826e2ada5fb9312201ed2a2c361b717bd6edb0960535b7cd892dd7b220f32821", "input_tokens": 237, "server_ms": 34.89}, {"id": "242df59a71ae30abc744", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.902604966424406, "probs": [0.9658152505243343, 0.014433297176003043, 0.01975145229966267], "top": "contradicted", "option_logits": null, "prompt_sha256": "532212397acf506bb6d051b3a006393dc8ef85bda014f19203862ce81f79cb6e", "input_tokens": 258, "server_ms": 35.7}, {"id": "65d53bafebb50d9273a4", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.74748202646151, "probs": [0.12983423759423549, 0.06933017382675004, 0.8008355885790144], "top": "prohibited", "option_logits": null, "prompt_sha256": "1f8361a1e21ef4b743dd9c9e598fc79fcb4580f603fffa095624a02a7f871639", "input_tokens": 233, "server_ms": 34.87}, {"id": "4c5cb6822a27c029976a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.31006500637159, "probs": [0.4766112692621149, 0.4766112692621149, 0.04677746147577021], "top": "prohibited", "option_logits": null, "prompt_sha256": "af4f9b8b19ba21529fec0312a655022cf6a87cc523574846a8a989ee1990d7c6", "input_tokens": 244, "server_ms": 35.21}, {"id": "fbfc0c5ab92dbb5b7638", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.32746495353058, "probs": [0.7814681688441721, 0.18460239897995706, 0.03392943217587092], "top": "prohibited", "option_logits": null, "prompt_sha256": "a3c809d72a52b805f6462ee41d98979e4ae2220ec982d280983372b48438bcc8", "input_tokens": 265, "server_ms": 35.57}, {"id": "84bdb2d001c7e024f8d8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.956023999489844, "probs": [0.040946062905879925, 0.943076329528072, 0.01597760756604808], "top": "A", "option_logits": null, "prompt_sha256": "feb22419e141b294675943aeffb02881cee059d214b759f7474c923eae50ae0f", "input_tokens": 222, "server_ms": 35.16}, {"id": "9e00994bdec725f3dba5", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.45648300135508, "probs": [0.08284676383042183, 0.8987728386104842, 0.018380397559094], "top": "A", "option_logits": null, "prompt_sha256": "aedc18bdfb6759f9bf4acc7dc32c42d7e7e0725516dfe4da61b13d514a81440d", "input_tokens": 233, "server_ms": 35.15}, {"id": "ad063020bdde29193686", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.81888197734952, "probs": [0.04127479806811213, 0.9506478143624966, 0.008077387569391328], "top": "A", "option_logits": null, "prompt_sha256": "db0ed7061b4d3486870324caa04d1ab2cf639ec6746cb46dc739d6fe10911e0a", "input_tokens": 254, "server_ms": 35.47}, {"id": "797d14fb61b9c31b2a73", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.8202149909921, "probs": [0.04589858171137419, 0.021619165778704597, 0.9324822525099211], "top": "contradicted", "option_logits": null, "prompt_sha256": "7f6ad652fb1432944343134fd29219b0907b5daf20b40e3fc1d4f2b10f8bc642", "input_tokens": 223, "server_ms": 35.21}, {"id": "b7848a79b779fa6213a6", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.82700799964368, "probs": [0.9168368722526529, 0.03511267860935454, 0.048050449137992716], "top": "contradicted", "option_logits": null, "prompt_sha256": "4dda01f8e5a06b0f40b7f85648cbcd7deef17266f90d19a6bcfb95567ee90ff7", "input_tokens": 234, "server_ms": 35.41}, {"id": "2b41fd051743512b14f8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.668488981202245, "probs": [0.9433379476711226, 0.03186734083364, 0.0247947114952375], "top": "contradicted", "option_logits": null, "prompt_sha256": "53751b75d7105e60ea0ab9350d45b6b2abae86831e385dd218165e10bb7d6230", "input_tokens": 255, "server_ms": 35.44}, {"id": "2355e01207ae3ee96f58", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.60645499220118, "probs": [0.8636555432040784, 0.10894307953240069, 0.0274013772635208], "top": "prohibited", "option_logits": null, "prompt_sha256": "1c3aabb10ad7afee2251ef1d9ea23150677c2f25bb6bcbf1080582d758fcefed", "input_tokens": 224, "server_ms": 35.14}, {"id": "7eeb4266198f316747bd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.10464004287496, "probs": [0.11834895829730337, 0.10439292369866547, 0.7772581180040311], "top": "prohibited", "option_logits": null, "prompt_sha256": "def5d5ec513a50260221d14cac2d3aba5db7999a5e075075167fb4114dc4534e", "input_tokens": 235, "server_ms": 36.19}, {"id": "dc45f218e1da88dcefe3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.01733400160447, "probs": [0.06048782007179681, 0.041513347298601795, 0.8979988326296015], "top": "prohibited", "option_logits": null, "prompt_sha256": "e2ed50b82d0ec1fb8f6f5942c730e1c84f5692ae10789f1e0854a01f38a69f61", "input_tokens": 256, "server_ms": 35.87}, {"id": "420328119f73acbcc528", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.831590979825705, "probs": [0.019926833275717554, 0.9743911251721546, 0.00568204155212776], "top": "A", "option_logits": null, "prompt_sha256": "12daa69ae333e1597e9b60b206d89e2eb8b1ec47d8f211922230bf4557252dfa", "input_tokens": 227, "server_ms": 35.23}, {"id": "e6d9003f307ac0c06184", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 62.449919001664966, "probs": [0.034594390781151056, 0.961790472870542, 0.003615136348306884], "top": "A", "option_logits": null, "prompt_sha256": "5ddc35af904973833090108b78c5be1652ebf2b4d6db97e185f5cc8b97a5ae58", "input_tokens": 238, "server_ms": 38.58}, {"id": "d89ff26feee2aab0c674", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.87371696997434, "probs": [0.021256974963064182, 0.9762246880063512, 0.002518337030584648], "top": "A", "option_logits": null, "prompt_sha256": "c5b878f4447eec2a58428ee9e5eebaa74113c1b4ead562bd74c502fabe3bda2d", "input_tokens": 259, "server_ms": 36.27}, {"id": "994e845d231fbbc1b314", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.66175400558859, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "bc9cc2077542b21bdd9d8cd4e359789d1f50f453f431bdcc559a121fd10005ff", "input_tokens": 227, "server_ms": 35.81}, {"id": "4a962c2278e610119138", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.302300966810435, "probs": [0.9180938798724528, 0.027357236358260754, 0.05454888376928654], "top": "contradicted", "option_logits": null, "prompt_sha256": "137dca1e1c43c32516a5166a102ffe279998867ea21f5c07a4da6fe923fdaf91", "input_tokens": 238, "server_ms": 35.39}, {"id": "1b55d33eb5abc7719a4a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.96245000883937, "probs": [0.9045670629095636, 0.03055760208149954, 0.06487533500893698], "top": "contradicted", "option_logits": null, "prompt_sha256": "a45798a5ed44d8c3daa136f8fbd76e1b129a5e0daa273feed8e69057b56c0f67", "input_tokens": 259, "server_ms": 35.85}, {"id": "6ed6a1f9c601b82a11d9", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.58436599979177, "probs": [0.35217658482883907, 0.48194107585431695, 0.16588233931684393], "top": "prohibited", "option_logits": null, "prompt_sha256": "3336984071b267a251457bd5f524c8d305f32ee35ce4934950f51e43d63108be", "input_tokens": 227, "server_ms": 35.26}, {"id": "bbc7a7fab955884da496", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.20570303592831, "probs": [0.4635360256308575, 0.21833490956880905, 0.3181290648003335], "top": "insufficient", "option_logits": null, "prompt_sha256": "43a24b5b5c514e8d69ab0aeb9f0c9bd466217b6f3dd887433e69b975bff9ebae", "input_tokens": 238, "server_ms": 35.41}, {"id": "fed61559ec517062e424", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.22693701600656, "probs": [0.11151623665779031, 0.4720761440937357, 0.41640761924847397], "top": "prohibited", "option_logits": null, "prompt_sha256": "4c8c1b9440887519f67da645f70ba2220f17517ab61dfc6bd4270bd723e39123", "input_tokens": 259, "server_ms": 35.57}, {"id": "854da63689519c5eeffe", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 31504.224194970448, "probs": [0.48418377207007135, 0.06107582365602279, 0.45474040427390583], "top": "B", "option_logits": null, "prompt_sha256": "de215717ec54a886a97496bd2b9e83313fe5175fb273c6e1533189b836473d51", "input_tokens": 221, "server_ms": 45.61}, {"id": "cbebaf6e14bfd85ee672", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.071258997078985, "probs": [0.35661738472173865, 0.05430022843256024, 0.5890823868457012], "top": "B", "option_logits": null, "prompt_sha256": "022b28511c8cc50945033d50b616eea1f6f426466cd3d42e3ed09a85ba2f95ed", "input_tokens": 232, "server_ms": 35.55}, {"id": "507f126320b35c49b3b3", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09/stability", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.62093501025811, "probs": [0.4030721610082638, 0.07888033800004987, 0.5180475009916863], "top": "B", "option_logits": null, "prompt_sha256": "dfd0ff134fde602c1579e17f1fb7d2fccc71940590d7cfd9382e3046688491f1", "input_tokens": 253, "server_ms": 35.11}, {"id": "21557955028145701b15", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.33757394691929, "probs": [0.032239858296844184, 0.013394898301201897, 0.9543652434019538], "top": "contradicted", "option_logits": null, "prompt_sha256": "61db8d7715d4b740ae99ffcd021a61cfe67f4750d706d361dee60d6b9b5ed1ad", "input_tokens": 213, "server_ms": 35.18}, {"id": "050a4186a02b28f0dccc", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.043329029809684, "probs": [0.9689224300153801, 0.017478355996226505, 0.013599213988393354], "top": "contradicted", "option_logits": null, "prompt_sha256": "19aba67242e5829cde2173541d3c3f561107e88fa7cabcdec19d89887a318d25", "input_tokens": 224, "server_ms": 35.23}, {"id": "aeb5013da56a9dd53665", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.77245502313599, "probs": [0.9722240772966334, 0.016471429008352915, 0.01130449369501362], "top": "contradicted", "option_logits": null, "prompt_sha256": "a72da482fd3d2416b5f34f5c2dae080982c4be80cd52e1501ae43e753184785e", "input_tokens": 245, "server_ms": 35.28}, {"id": "9cb7a576fead7fc845ab", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.121744993608445, "probs": [0.15440673380738387, 0.056587318882760376, 0.7890059473098556], "top": "prohibited", "option_logits": null, "prompt_sha256": "e7159b85cfbfec17a6e43ab69db8f2983a32d730d1982f40bda57998f17afb7f", "input_tokens": 218, "server_ms": 35.02}, {"id": "ddd254fae8e415615a8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.74640696775168, "probs": [0.8008355885790144, 0.06933017382675004, 0.12983423759423549], "top": "prohibited", "option_logits": null, "prompt_sha256": "f3402ab95b607129a715e772b948fc32ce7dd704327f8c2b535d13ad151a0f3b", "input_tokens": 229, "server_ms": 35.29}, {"id": "2b51ed50ef91d14bfb1e", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.33874798472971, "probs": [0.9199580828054892, 0.037513419983045006, 0.042528497211465836], "top": "prohibited", "option_logits": null, "prompt_sha256": "2aa63b739420a25f5fc5fec212633fbf1a7248fe59a2848f57c1c7e71f15030e", "input_tokens": 250, "server_ms": 35.49}, {"id": "65b87e9b46818cc758c0", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.0363759608008, "probs": [0.9903558898353046, 0.0054239502826090285, 0.004220159882086359], "top": "A", "option_logits": null, "prompt_sha256": "9eb2e6c380378dc67d0fa371aa47cc6dc05ab89290504f96395cb9dd9feaecf4", "input_tokens": 206, "server_ms": 35.16}, {"id": "d39d7fc35c42bb898ea8", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.85755602037534, "probs": [0.01953747359501767, 0.02511048931910777, 0.9553520370858747], "top": "A", "option_logits": null, "prompt_sha256": "a957e8d1f5e9078ec977f3f04f839dfad11ec1b7847e82a713188f1567caee7a", "input_tokens": 217, "server_ms": 35.07}, {"id": "61ef89ceb2a564f24805", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.05425897333771, "probs": [0.02507876464955978, 0.02077619807256723, 0.9541450372778729], "top": "A", "option_logits": null, "prompt_sha256": "c4e1a6441206a29298affafa7769a26b61a288f0a0f3651f78ebb149a8392d7c", "input_tokens": 238, "server_ms": 35.0}, {"id": "2bdff68fef60c15b8929", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": null, "ok": true, "e2e_ms": 68.22072895010933, "probs": [0.6860736102625705, 0.2677135270296671, 0.046212862707762405], "top": "contradicted", "option_logits": null, "prompt_sha256": "e964de06c4969c253d00f92a787b6ea8ecab26543bca93b98e9f52974495df1e", "input_tokens": 224, "server_ms": 35.52}, {"id": "d727a02607d044abafed", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.833508969750255, "probs": [0.0913398978679528, 0.41170029433936683, 0.4969598077926804], "top": "contradicted", "option_logits": null, "prompt_sha256": "6610f7db26b88162c5f67c24fa915dcf9472a6eccab1a2bb9a23d1a13a633e10", "input_tokens": 235, "server_ms": 35.31}, {"id": "c89cb1e57ed5b22d794b", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d/stability", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.74913499690592, "probs": [0.07756313123884137, 0.225345624605489, 0.6970912441556697], "top": "contradicted", "option_logits": null, "prompt_sha256": "e3f03242dda72583eedf51d0fe3306890a6d7edc3a056552be9ce6d4f9863d9f", "input_tokens": 256, "server_ms": 34.99}, {"id": "b1521d74fb803e551fdd", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.17347699403763, "probs": [0.22212817687379355, 0.5692513354541036, 0.2086204876721029], "top": "permitted", "option_logits": null, "prompt_sha256": "abb84ccebbde07e59ac0884f0d7a36011f00554e97c15fa4f631722630326b24", "input_tokens": 226, "server_ms": 35.21}, {"id": "3117510a0d5b768c4096", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.89424896566197, "probs": [0.1225262494395239, 0.7098009505679745, 0.1676727999925016], "top": "permitted", "option_logits": null, "prompt_sha256": "35340572a7bf8582ad7d4e19ac361f97f252504d1459f63d4cd066366b844130", "input_tokens": 237, "server_ms": 34.85}, {"id": "90632d8d2066bd79c796", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee/stability", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.3972909799777, "probs": [0.15419029023068853, 0.7399874919908419, 0.10582221777846951], "top": "permitted", "option_logits": null, "prompt_sha256": "5528b7f79def91ae1261709fabab22dc704879942bd5055ac46abbd155f078ad", "input_tokens": 258, "server_ms": 35.49}, {"id": "ffb51cef94e857eb8c8a", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.0042279753834, "probs": [0.03874679809393158, 0.9502047376512637, 0.011048464254804755], "top": "A", "option_logits": null, "prompt_sha256": "04f96aa3570fc40ee909f0dcce87caa80677a6b8b3ae68c021f79cf181894895", "input_tokens": 213, "server_ms": 34.94}, {"id": "a34e8187417ab3b52f30", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.53316699620336, "probs": [0.07883110016310216, 0.9105811569450533, 0.01058774289184458], "top": "A", "option_logits": null, "prompt_sha256": "0e594218d2864709f362a3b24fe6bcaf39dda0a356b626b6eed74d60eb0c4f73", "input_tokens": 224, "server_ms": 35.31}, {"id": "31f53baf67e4cb25aeff", "set": "perturbations108", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3/stability", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.400627977680415, "probs": [0.1375016638769001, 0.8481293219425575, 0.014369014180542438], "top": "A", "option_logits": null, "prompt_sha256": "985c820701cd08c4e36c2e35ad09b5cd585eb3d61defb1a3c18ee87cec2af703", "input_tokens": 245, "server_ms": 34.91}, {"id": "cicada-gate-w1-now/lights-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.15224004536867, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "37bb270e126485476866e974e30184d13fb2b8d75d63b4d09db2f15a4f16cd4c", "input_tokens": 257, "server_ms": 36.0}, {"id": "cicada-gate-w1-now/lights-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.43011295190081, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.87}, {"id": "cicada-gate-w1-now/timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.55448599625379, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "16c086bc994a85e0432805c5e9fab68d2980ba7d7f60d325bc1c61b25c292d37", "input_tokens": 258, "server_ms": 35.86}, {"id": "cicada-gate-w1-now/timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.151971988379955, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.03}, {"id": "cicada-gate-w1-now/weather/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.15390298003331, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "90ab7cd6cf2490aca31ce24cab1d5ecb2142a95b2750ff13ee896f9ffef6b811", "input_tokens": 257, "server_ms": 35.51}, {"id": "cicada-gate-w1-now/weather/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.5716749727726, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.17}, {"id": "cicada-gate-w1-now/weather-followup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.563647019676864, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "122d18d305083b090ff1de5d0587544e0926f2969d6c47be07552baf190830a7", "input_tokens": 254, "server_ms": 35.58}, {"id": "cicada-gate-w1-now/weather-followup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.04691399214789, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.95}, {"id": "cicada-gate-w1-now/add-milk/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.88213903643191, "probs": [0.12511350509772579, 0.8748864949022742], "top": "ordinary", "option_logits": null, "prompt_sha256": "d1d87fb2eda410fcbdbbb9f2550b3b3ac1be756e49903e6e3934118aae58bc9b", "input_tokens": 258, "server_ms": 35.64}, {"id": "cicada-gate-w1-now/add-milk/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.76546097593382, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.94}, {"id": "cicada-gate-w1-now/thermostat/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.40507096843794, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "8a9a9908d29f78ef241e5d958b2463b0086ca664d5d317d16ce74e11c813b2a7", "input_tokens": 259, "server_ms": 35.52}, {"id": "cicada-gate-w1-now/thermostat/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.420110954903066, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.15}, {"id": "cicada-gate-w1-now/what-time/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.73559502186254, "probs": [0.08937422170979005, 0.91062577829021], "top": "ordinary", "option_logits": null, "prompt_sha256": "f393dac77e240d1fc3e87685205fa16d3d090fc847b3706bc2b473adf98e9411", "input_tokens": 256, "server_ms": 35.17}, {"id": "cicada-gate-w1-now/what-time/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.939377957955, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.96}, {"id": "cicada-gate-w1-now/tablespoons/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.51377202896401, "probs": [0.08439803417650704, 0.9156019658234928], "top": "ordinary", "option_logits": null, "prompt_sha256": "eac64b86575d2cfefdd93b69192342caa85f53c8c1f2ce99a049693b8c0ec75e", "input_tokens": 258, "server_ms": 35.57}, {"id": "cicada-gate-w1-now/tablespoons/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.75796400383115, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.89}, {"id": "cicada-gate-w1-now/times-table/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.060466994997114, "probs": [0.09461348765523125, 0.9053865123447689], "top": "ordinary", "option_logits": null, "prompt_sha256": "c2b4411ed1b9a0655946a1cf96a9f4395b4723f305f9d4def1ab7e2d57e7b037", "input_tokens": 257, "server_ms": 35.39}, {"id": "cicada-gate-w1-now/times-table/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.738154005259275, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.25}, {"id": "cicada-gate-w1-now/recipe-step/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.99836698779836, "probs": [0.15526082656247686, 0.844739173437523], "top": "ordinary", "option_logits": null, "prompt_sha256": "ae87f53ef3e34104b9548907ac3e4914604ee7905e769aca2dec341838eb26f8", "input_tokens": 261, "server_ms": 35.78}, {"id": "cicada-gate-w1-now/recipe-step/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.51901304675266, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/soup/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.954943967051804, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "ed626571ee8b8a04454305a0f54e9b70ee6a83bae0f05e562d087ca0331a9303", "input_tokens": 259, "server_ms": 35.39}, {"id": "cicada-gate-w1-now/soup/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.61925903707743, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.19}, {"id": "cicada-gate-w1-now/play-music/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.79648896632716, "probs": [0.2681939143399331, 0.7318060856600669], "top": "ordinary", "option_logits": null, "prompt_sha256": "4c22b161ee967bef478c2d72c7a103f3f69af9fab7e84f7c4b3c9178af2ba260", "input_tokens": 255, "server_ms": 35.51}, {"id": "cicada-gate-w1-now/play-music/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.06061200099066, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.8}, {"id": "cicada-gate-w1-now/dentist/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.058987996540964, "probs": [0.10012611628383591, 0.8998738837161642], "top": "ordinary", "option_logits": null, "prompt_sha256": "185a1e918f09474cebb97fed81f2ed02f1e15cd7fcb418f704676c9a7d8dd863", "input_tokens": 257, "server_ms": 35.59}, {"id": "cicada-gate-w1-now/dentist/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.342335999943316, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 37.58}, {"id": "cicada-gate-w1-now/spell/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.093940045684576, "probs": [0.10592235395880453, 0.8940776460411956], "top": "ordinary", "option_logits": null, "prompt_sha256": "aadfdbfdb6e924158987d74d8abf46b50a91c22386567aa68e8ebe3cdac38b60", "input_tokens": 257, "server_ms": 36.29}, {"id": "cicada-gate-w1-now/spell/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.83912398247048, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.97}, {"id": "cicada-gate-w1-now/remind/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.904529999475926, "probs": [0.11840617139815589, 0.8815938286018441], "top": "ordinary", "option_logits": null, "prompt_sha256": "af282ab0ddb7cdeb8f5bfd51b74ec87e62a1cb93af6034412016ae1367abd1e8", "input_tokens": 260, "server_ms": 35.57}, {"id": "cicada-gate-w1-now/remind/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.555169026367366, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.8}, {"id": "cicada-gate-w1-now/volume/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.94639301812276, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "6958736e40be6c6078b4d5c7088dc2ce294b752caf980ad484a11f104d1bc4cf", "input_tokens": 257, "server_ms": 35.61}, {"id": "cicada-gate-w1-now/volume/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.24740899959579, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.94}, {"id": "cicada-gate-w1-now/dog-died/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.234048002399504, "probs": [0.9727760393154234, 0.027223960684576436], "top": "react", "option_logits": null, "prompt_sha256": "2733eb533f2758c0b9660f7d8d3474d1950de14bf80b36b7b35f1356457f3c4f", "input_tokens": 257, "server_ms": 35.58}, {"id": "cicada-gate-w1-now/dog-died/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.25271899346262, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.76}, {"id": "cicada-gate-w1-now/got-job/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.66400599107146, "probs": [0.9710643566281367, 0.02893564337186314], "top": "react", "option_logits": null, "prompt_sha256": "24ed6113b661626913f245d080ef749553d24514dfdcd1f4da29284af28ba2ed", "input_tokens": 264, "server_ms": 35.57}, {"id": "cicada-gate-w1-now/got-job/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.62169002182782, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.24}, {"id": "cicada-gate-w1-now/intruder/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.086261021438986, "probs": [0.9290542163538821, 0.07094578364611796], "top": "react", "option_logits": null, "prompt_sha256": "b43b54e9b069684793b90353161a47d5fdc8e8103ff14fcd2778ca8e02124816", "input_tokens": 264, "server_ms": 35.57}, {"id": "cicada-gate-w1-now/intruder/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w1-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.58364398125559, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.19}, {"id": "cicada-gate-w1-now/fish/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.31825400935486, "probs": [0.8275619327448428, 0.1724380672551571], "top": "react", "option_logits": null, "prompt_sha256": "e485800493ea1e02dfaf153c2a8a6d5f0f788948a3a006964f6ea6020c0a1d52", "input_tokens": 264, "server_ms": 35.58}, {"id": "cicada-gate-w1-now/fish/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.21929801721126, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.07}, {"id": "cicada-gate-w1-now/hospital/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.77178295748308, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "802d688ac6ecd38257197c8a642e92401071004b837cb227e0559cb07ea0960e", "input_tokens": 259, "server_ms": 35.42}, {"id": "cicada-gate-w1-now/hospital/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.21156099671498, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.99}, {"id": "cicada-gate-w1-now/laid-off/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.47985198022798, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "bdd688b377e4dc12be8b31692de83628d6c991c8327729b6464bab0a7d244e67", "input_tokens": 257, "server_ms": 35.41}, {"id": "cicada-gate-w1-now/laid-off/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.68876300053671, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.86}, {"id": "cicada-gate-w1-now/baby-steps/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.64557302184403, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "fca4092ad2f665575a3df9ca9f6014362c60511bbe0c444548a3fbaf4c416124", "input_tokens": 259, "server_ms": 35.52}, {"id": "cicada-gate-w1-now/baby-steps/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.23034699540585, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.82}, {"id": "cicada-gate-w1-now/cut-finger/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.298118989914656, "probs": [0.9471470189565372, 0.052852981043462664], "top": "react", "option_logits": null, "prompt_sha256": "4ba9f9f33588b9b9b9db4f40691795adad8eaf490e5cef7993a1613581ddb2cb", "input_tokens": 262, "server_ms": 35.55}, {"id": "cicada-gate-w1-now/cut-finger/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.834794973023236, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.81}, {"id": "cicada-gate-w1-now/raccoon/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.820647997781634, "probs": [0.8998738837161642, 0.10012611628383591], "top": "react", "option_logits": null, "prompt_sha256": "a46a5804e163de7f4aff420c873f9f457185a5113defa26200bffa7815a743cf", "input_tokens": 262, "server_ms": 35.45}, {"id": "cicada-gate-w1-now/raccoon/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.292245022021234, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.78}, {"id": "cicada-gate-w1-now/snow-june/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.76008396362886, "probs": [0.91062577829021, 0.08937422170979005], "top": "react", "option_logits": null, "prompt_sha256": "56f00d8abefba9014cd351fc0b504aed1511773bea6630721c55e0fd627f680d", "input_tokens": 260, "server_ms": 35.71}, {"id": "cicada-gate-w1-now/snow-june/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.71055200137198, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.89}, {"id": "cicada-gate-w1-now/toaster/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.72095498023555, "probs": [0.608060011110835, 0.39193998888916504], "top": "react", "option_logits": null, "prompt_sha256": "2e3f9dfc2053516e1fffacb0378398ffe661f0eb1d84c856d934e399cb42dba6", "input_tokens": 262, "server_ms": 35.54}, {"id": "cicada-gate-w1-now/toaster/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.234460996463895, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.92}, {"id": "cicada-gate-w1-now/scarecrow/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.9143499918282, "probs": [0.7064759712765732, 0.2935240287234267], "top": "react", "option_logits": null, "prompt_sha256": "c65a7f7de08b04f40a292ca857711bddf306a68b5effa8c23bce621acfc61dc7", "input_tokens": 268, "server_ms": 35.71}, {"id": "cicada-gate-w1-now/scarecrow/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.538610994815826, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.76}, {"id": "cicada-gate-w1-now/cookie/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.682242054492235, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "a6e458e3306727fdc0f256d7f9ae4ec7879e1242fb8511151352b0df4e10b9a3", "input_tokens": 260, "server_ms": 35.59}, {"id": "cicada-gate-w1-now/cookie/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.70438699517399, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.84}, {"id": "cicada-gate-w1-now/useless/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.08637794991955, "probs": [0.9248060506082927, 0.07519394939170732], "top": "react", "option_logits": null, "prompt_sha256": "e452379f759f57b3618d4c4a7b2dd0300b51e6a2e3f70328d53e089cbbc5c23c", "input_tokens": 261, "server_ms": 35.56}, {"id": "cicada-gate-w1-now/useless/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.25661797542125, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.92}, {"id": "cicada-gate-w1-now/cheese/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.27113502426073, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "a225f3037b72c9434d234aa1615b7decb6002a5481972a98276770f6dd2ce1a4", "input_tokens": 258, "server_ms": 35.56}, {"id": "cicada-gate-w1-now/cheese/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w1-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.702790999319404, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.9}, {"id": "cicada-gate-w1-now/goodnight/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.23077994398773, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "69f47d42b7638a38bd04949f16304a9bd29f333cf62a3f0f430ebcff02e8ab63", "input_tokens": 257, "server_ms": 35.52}, {"id": "cicada-gate-w1-now/goodnight/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.66324301576242, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.09}, {"id": "cicada-gate-w1-now/thanks/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.78761700214818, "probs": [0.8940776460411956, 0.10592235395880453], "top": "react", "option_logits": null, "prompt_sha256": "cf88dd627ecf21f99ad2baf4f9142e9a20d57648e5fc16f1f067f9d5cb2c7a73", "input_tokens": 261, "server_ms": 35.69}, {"id": "cicada-gate-w1-now/thanks/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.68052901374176, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.83}, {"id": "cicada-gate-w1-now/sit-with-me/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.252022004220635, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "6d56dbfa4a01bb7d989ad4cb947aa0820a391289ac94c4f271084deabf3d65a3", "input_tokens": 258, "server_ms": 35.65}, {"id": "cicada-gate-w1-now/sit-with-me/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.06513496907428, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 34.86}, {"id": "cicada-gate-w1-now/stop-timer/gate", "set": "cicada-w1", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.81724500330165, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "ba70cee81d8ea6a7933b7f6da981f5df76667aa494b05924c6aa5d3ece2b71a8", "input_tokens": 255, "server_ms": 35.2}, {"id": "cicada-gate-w1-now/stop-timer/gate#blind", "set": "cicada-w1", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w1-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.36097499821335, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "9b6e6fd970e900f77797f93c6d8ad54b676b27f73afcb840490e7ddc81b92b2f", "input_tokens": 250, "server_ms": 35.0}, {"id": "cicada-gate-w2-now/lights-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.85086197545752, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "ad0c2b593b906cdc0ff3581e7bee6fc3f71f505805a0f8defa0ea475de0dc4e0", "input_tokens": 184, "server_ms": 35.31}, {"id": "cicada-gate-w2-now/lights-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/lights-off", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.36743802763522, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.3}, {"id": "cicada-gate-w2-now/timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.17682400345802, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e2cb59142e602052b8c679629180e388c0a982fc08fc99ce3d4201cc31543be", "input_tokens": 185, "server_ms": 35.2}, {"id": "cicada-gate-w2-now/timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/timer", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.71506099961698, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.95}, {"id": "cicada-gate-w2-now/weather/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.98149801092222, "probs": [0.1636682776269396, 0.8363317223730603], "top": "ordinary", "option_logits": null, "prompt_sha256": "86636ec948054d8d9db67d54941cf9162c8437f61e792c5aa057609c460e1314", "input_tokens": 184, "server_ms": 35.39}, {"id": "cicada-gate-w2-now/weather/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.282027947716415, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.15}, {"id": "cicada-gate-w2-now/weather-followup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.2490509795025, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "e4c73f8f673df37900974014f441f4b97c84eb00a75e7652bb557de32ca18fd9", "input_tokens": 181, "server_ms": 35.09}, {"id": "cicada-gate-w2-now/weather-followup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/weather-followup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.57614603033289, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.97}, {"id": "cicada-gate-w2-now/add-milk/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.885892044287175, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "76d6dbfee4b1e86e057a6ac6dfc9faf2957eacc1472c5b75ac62077217e8ffbb", "input_tokens": 185, "server_ms": 35.0}, {"id": "cicada-gate-w2-now/add-milk/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/add-milk", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.525170020293444, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.02}, {"id": "cicada-gate-w2-now/thermostat/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.45083394786343, "probs": [0.45308504361568214, 0.5469149563843178], "top": "ordinary", "option_logits": null, "prompt_sha256": "9e52653975fb258134369efc6fb8f79f87b571c93fdd7792c1f160c362818880", "input_tokens": 186, "server_ms": 35.23}, {"id": "cicada-gate-w2-now/thermostat/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/thermostat", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.627118992153555, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.84}, {"id": "cicada-gate-w2-now/what-time/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.43715997878462, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "d8fe04845d52c73989d7d686ba336e8a037ae0bfa9da79ba697267ab9fc9e1c7", "input_tokens": 183, "server_ms": 35.34}, {"id": "cicada-gate-w2-now/what-time/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/what-time", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.52793802740052, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.83}, {"id": "cicada-gate-w2-now/tablespoons/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.272554978728294, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "b80acb508ebe1b7bcd739b7d309f8f87d5c20e0ef5c2232f241b7480c484e7c0", "input_tokens": 185, "server_ms": 35.18}, {"id": "cicada-gate-w2-now/tablespoons/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/tablespoons", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.005026002414525, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.83}, {"id": "cicada-gate-w2-now/times-table/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.80769100924954, "probs": [0.14720923592665844, 0.8527907640733415], "top": "ordinary", "option_logits": null, "prompt_sha256": "f32b701cc11fb60558808c0e63800242f9df430a698ba91b1e292c29a97ff073", "input_tokens": 184, "server_ms": 35.27}, {"id": "cicada-gate-w2-now/times-table/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/times-table", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.563431021757424, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.79}, {"id": "cicada-gate-w2-now/recipe-step/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.268597985152155, "probs": [0.1910858322004182, 0.8089141677995819], "top": "ordinary", "option_logits": null, "prompt_sha256": "6f3741b23c671b7677beef8b001c3a339acba3443d895c0ff7a9366672072267", "input_tokens": 188, "server_ms": 35.26}, {"id": "cicada-gate-w2-now/recipe-step/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/recipe-step", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.01768402568996, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.69}, {"id": "cicada-gate-w2-now/soup/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.641787031665444, "probs": [0.1724380672551571, 0.8275619327448428], "top": "ordinary", "option_logits": null, "prompt_sha256": "15beeee51e76a84fe517ac3b2d63a863db143e88ec0691ea434b1f949a713273", "input_tokens": 186, "server_ms": 35.17}, {"id": "cicada-gate-w2-now/soup/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/soup", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.502100996207446, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.97}, {"id": "cicada-gate-w2-now/play-music/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.212903963867575, "probs": [0.3340072352673295, 0.6659927647326706], "top": "ordinary", "option_logits": null, "prompt_sha256": "3013ff02e1dda210c90f8b5bb0c0b7fed227b12d2f9a839445087869339e0294", "input_tokens": 182, "server_ms": 35.44}, {"id": "cicada-gate-w2-now/play-music/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/play-music", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.58846398303285, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.95}, {"id": "cicada-gate-w2-now/dentist/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.440027044620365, "probs": [0.20097168954465905, 0.799028310455341], "top": "ordinary", "option_logits": null, "prompt_sha256": "6ed8659f14076b05a2f24964056158993d775693258fb0bcb9c272da6a2f29bf", "input_tokens": 184, "server_ms": 35.07}, {"id": "cicada-gate-w2-now/dentist/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/dentist", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.21973696118221, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.81}, {"id": "cicada-gate-w2-now/spell/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.633466945029795, "probs": [0.13214384255458153, 0.8678561574454184], "top": "ordinary", "option_logits": null, "prompt_sha256": "7b1a321825d1994784a0ece39c659d7cb2ac3e5c425bfb094b70b4958ffb1702", "input_tokens": 184, "server_ms": 34.99}, {"id": "cicada-gate-w2-now/spell/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/spell", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.48581599537283, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.81}, {"id": "cicada-gate-w2-now/remind/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.22263500420377, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "48b2dcdcd628051de6b35a16985c4ee1ffa062d521f412bb838819f7b5eabd7a", "input_tokens": 187, "server_ms": 35.57}, {"id": "cicada-gate-w2-now/remind/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/remind", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.19493298884481, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.82}, {"id": "cicada-gate-w2-now/volume/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.25060401810333, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "769f39cd90add71a604bfb344dc4c809cf38fa4794fdb93b2835ec172aafa3ca", "input_tokens": 184, "server_ms": 35.11}, {"id": "cicada-gate-w2-now/volume/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/volume", "gold": ["ordinary"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.976471011992544, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.26}, {"id": "cicada-gate-w2-now/dog-died/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.80881200451404, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "80a047853997be9d216e0a21a7f7ac20592c915d66e4ae372c47c978fa03ddfe", "input_tokens": 184, "server_ms": 35.07}, {"id": "cicada-gate-w2-now/dog-died/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/dog-died", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.824401995167136, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.84}, {"id": "cicada-gate-w2-now/got-job/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.228223976679146, "probs": [0.7439389554145914, 0.25606104458540857], "top": "react", "option_logits": null, "prompt_sha256": "2f3dd4c049a82b3904d7dd2258746905a11e5d1e7f74607db135b844df32b91e", "input_tokens": 191, "server_ms": 35.28}, {"id": "cicada-gate-w2-now/got-job/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/got-job", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.585360990837216, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.96}, {"id": "cicada-gate-w2-now/intruder/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.302978980354965, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "d473bbe6e3a042a4189f1dd15fbd7b9fb82d12fd93c7b9f0d852093e0eee67ea", "input_tokens": 191, "server_ms": 35.19}, {"id": "cicada-gate-w2-now/intruder/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "control", "group": "cicada-gate-w2-now/intruder", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.404442965984344, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.14}, {"id": "cicada-gate-w2-now/fish/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.096336021553725, "probs": [0.30670018569615076, 0.6932998143038492], "top": "ordinary", "option_logits": null, "prompt_sha256": "e016e8f87924bd6367f0af1e794897be0bb6efe2da1793366c3c34b632748a8e", "input_tokens": 191, "server_ms": 35.07}, {"id": "cicada-gate-w2-now/fish/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/fish", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.82746098237112, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.0}, {"id": "cicada-gate-w2-now/hospital/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.038333950098604, "probs": [0.4843206825122851, 0.5156793174877148], "top": "ordinary", "option_logits": null, "prompt_sha256": "194e72ad6a3240e94296f4c1304e3acfdd224b51173def77329ac471a6770153", "input_tokens": 186, "server_ms": 35.17}, {"id": "cicada-gate-w2-now/hospital/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/hospital", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.539310010615736, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.74}, {"id": "cicada-gate-w2-now/laid-off/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.54932299815118, "probs": [0.5313278281391198, 0.4686721718608801], "top": "react", "option_logits": null, "prompt_sha256": "a95fd762372f5a6984361502a0c946aff43151e832de42811e64b8e78ee963a7", "input_tokens": 184, "server_ms": 35.05}, {"id": "cicada-gate-w2-now/laid-off/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/laid-off", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.85695201018825, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.86}, {"id": "cicada-gate-w2-now/baby-steps/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.87044000579044, "probs": [0.6798002499763626, 0.3201997500236374], "top": "react", "option_logits": null, "prompt_sha256": "5cb010480b479f161d0aad57f4ab3e0769bb15c880feacaad19d950ab50fd220", "input_tokens": 186, "server_ms": 35.0}, {"id": "cicada-gate-w2-now/baby-steps/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/baby-steps", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.54728503152728, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.9}, {"id": "cicada-gate-w2-now/cut-finger/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.54747304227203, "probs": [0.5156793174877148, 0.4843206825122851], "top": "react", "option_logits": null, "prompt_sha256": "c863f24bc9418b799a6d6c2044bf07374d1fcc42f905915d2198fd53f275c921", "input_tokens": 189, "server_ms": 35.4}, {"id": "cicada-gate-w2-now/cut-finger/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cut-finger", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.514181985519826, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.89}, {"id": "cicada-gate-w2-now/raccoon/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.495767002459615, "probs": [0.4069889545588398, 0.5930110454411601], "top": "ordinary", "option_logits": null, "prompt_sha256": "1a426b0b7eebc96e3557b66413d48b938a17ed2fd5370f5bd165d9a11cef8d9a", "input_tokens": 189, "server_ms": 35.13}, {"id": "cicada-gate-w2-now/raccoon/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/raccoon", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.23124100686982, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.9}, {"id": "cicada-gate-w2-now/snow-june/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.54274803539738, "probs": [0.42221451068097277, 0.5777854893190272], "top": "ordinary", "option_logits": null, "prompt_sha256": "35037fda6fe66b79a3717d191706a7f03eced03d2840e763e794ea1e1d159895", "input_tokens": 187, "server_ms": 35.24}, {"id": "cicada-gate-w2-now/snow-june/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/snow-june", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.377042000647634, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.81}, {"id": "cicada-gate-w2-now/toaster/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.57759696198627, "probs": [0.24429381869718142, 0.7557061813028185], "top": "ordinary", "option_logits": null, "prompt_sha256": "0de638423e8c5d5e95126a5b7dae73d8e757a1daece888bcad18912d32fac7b7", "input_tokens": 189, "server_ms": 35.38}, {"id": "cicada-gate-w2-now/toaster/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/toaster", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.86272605089471, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.88}, {"id": "cicada-gate-w2-now/scarecrow/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.71189396828413, "probs": [0.23289806532119295, 0.767101934678807], "top": "ordinary", "option_logits": null, "prompt_sha256": "2916b1e059434f4c17f48b63fd6446f83cb7481d19b50e6a90ddb42a82d85abe", "input_tokens": 195, "server_ms": 35.4}, {"id": "cicada-gate-w2-now/scarecrow/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/scarecrow", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.32620599074289, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.91}, {"id": "cicada-gate-w2-now/cookie/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.76435900386423, "probs": [0.25606104458540857, 0.7439389554145914], "top": "ordinary", "option_logits": null, "prompt_sha256": "de375bc98b339571beebb071e50719d9949124a175481fdd3845688dcdb04b97", "input_tokens": 187, "server_ms": 35.21}, {"id": "cicada-gate-w2-now/cookie/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cookie", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.12516598496586, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.98}, {"id": "cicada-gate-w2-now/useless/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.44579101074487, "probs": [0.18157574487480185, 0.8184242551251981], "top": "ordinary", "option_logits": null, "prompt_sha256": "205993715f2586a980060e1bb605d05af0f98854e5989af97a728f7ce11555f5", "input_tokens": 188, "server_ms": 35.06}, {"id": "cicada-gate-w2-now/useless/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/useless", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.80371900880709, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.86}, {"id": "cicada-gate-w2-now/cheese/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.88088298402727, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "d040ef49e06e994273da73faf9f7045d693fe5c52cdc213906a674526418c1d2", "input_tokens": 185, "server_ms": 35.01}, {"id": "cicada-gate-w2-now/cheese/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "case", "group": "cicada-gate-w2-now/cheese", "gold": ["react"], "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.980698994360864, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.06}, {"id": "cicada-gate-w2-now/goodnight/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.54228103533387, "probs": [0.2935240287234267, 0.7064759712765732], "top": "ordinary", "option_logits": null, "prompt_sha256": "c5f99b9eba8c8c61b2b1660ddd2f16bbffb148fcaeb1ab23a03f4f592a48a3b7", "input_tokens": 184, "server_ms": 35.09}, {"id": "cicada-gate-w2-now/goodnight/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/goodnight", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.06232499424368, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.01}, {"id": "cicada-gate-w2-now/thanks/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.37578698527068, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "13674b652950fc43a4f538f373eb82985e3bc9c04785d52c7546312abd74938e", "input_tokens": 188, "server_ms": 35.18}, {"id": "cicada-gate-w2-now/thanks/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/thanks", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.098600008524954, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 34.92}, {"id": "cicada-gate-w2-now/sit-with-me/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.42871299944818, "probs": [0.43758935794935333, 0.5624106420506467], "top": "ordinary", "option_logits": null, "prompt_sha256": "5a880ca783c35091cce278857e9b37f098ad7db31dbd294c6fcad0f55f47c7ca", "input_tokens": 185, "server_ms": 35.06}, {"id": "cicada-gate-w2-now/sit-with-me/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/sit-with-me", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.43243503011763, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.07}, {"id": "cicada-gate-w2-now/stop-timer/gate", "set": "cicada-w2", "cond": "evidence", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.38176997611299, "probs": [0.37709365395745764, 0.6229063460425425], "top": "ordinary", "option_logits": null, "prompt_sha256": "9c2a784ed4917379405889f124c83bd1670328e68774cee1f31a8ed48ab3aabb", "input_tokens": 182, "server_ms": 35.23}, {"id": "cicada-gate-w2-now/stop-timer/gate#blind", "set": "cicada-w2", "cond": "blind", "decision": "gate", "tag": "open", "group": "cicada-gate-w2-now/stop-timer", "gold": null, "forbid": [], "option_ids": ["react", "ordinary"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.71889396337792, "probs": [0.21123545993246395, 0.788764540067536], "top": "ordinary", "option_logits": null, "prompt_sha256": "42d890257870c87b0753d4dd9437f4ce422ac7e602c048b950b000b5f16d449e", "input_tokens": 177, "server_ms": 35.06}, {"id": "wyrd-scene-fletcher/f-idle-look/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.2026359946467, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "058da368fb54e53f1722bf167e6db03567ab88733dfda904dcfbdfa17b020096", "input_tokens": 349, "server_ms": 37.16}, {"id": "wyrd-scene-fletcher/f-idle-look/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.74836899479851, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.06}, {"id": "wyrd-scene-fletcher/f-idle-look/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.2454250282608, "probs": [0.7718190530995124, 0.182323036446684, 0.04585791045380365], "top": "stay", "option_logits": null, "prompt_sha256": "84c7f8435b2ab9a50ab5e03131ce3b11f93ecf83440dd76bad82255d6ca55c8f", "input_tokens": 402, "server_ms": 36.38}, {"id": "wyrd-scene-fletcher/f-idle-look/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.71112696593627, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.95}, {"id": "wyrd-scene-fletcher/f-idle-look/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.09115499444306, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "955c44e495e2a977cdc4d881d9c1fe4671ddf63577d50ea8831fb4fd2e27a202", "input_tokens": 362, "server_ms": 36.07}, {"id": "wyrd-scene-fletcher/f-idle-look/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.59550396678969, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.76}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.39117299485952, "probs": [0.9498122139379266, 0.0439086160356991, 0.006279170026374424], "top": "stay", "option_logits": null, "prompt_sha256": "73ea96cae6ae288e85508ee6b82f378c16f2f8249e90fc7e944729f0f1177bd8", "input_tokens": 414, "server_ms": 36.68}, {"id": "wyrd-scene-fletcher/f-idle-look/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-idle-look", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.517221997026354, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.27}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.48525998508558, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "3815ee0d0796f336f168e99d5aa788ece61208e45b945e01bf6637c92f4d3dc9", "input_tokens": 349, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.26245800824836, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.09}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.62468595942482, "probs": [0.7344793363943535, 0.0560873753680077, 0.20943328823763885], "top": "stay", "option_logits": null, "prompt_sha256": "9979fba74f296b7c56dd57196f7d52dea2221edd6b44b0ec6bb09b5f9ba07486", "input_tokens": 402, "server_ms": 36.05}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.067716013174504, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.91}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.58968598395586, "probs": [0.9692484548756632, 0.03075154512433679], "top": "same", "option_logits": null, "prompt_sha256": "b470826b04361cf78a4ee1be70f9c8e5060a40708b4c707e636ea653ec6186a5", "input_tokens": 362, "server_ms": 35.9}, {"id": "wyrd-scene-fletcher/f-plan-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.113378991838545, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.56}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.096509993541986, "probs": [0.9398897417986248, 0.0338066486561543, 0.026303609545220985], "top": "stay", "option_logits": null, "prompt_sha256": "3a6c6db96ecf98a2ac4d23c54a51eb817928adc810a43463165b9a5520252f74", "input_tokens": 414, "server_ms": 36.1}, {"id": "wyrd-scene-fletcher/f-plan-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-plan-tavern", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.02843498066068, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.13}, {"id": "wyrd-scene-fletcher/f-go-tavern/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.38917700666934, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "b9fe8e915d19e7f984dea44651b50f5154d57c30fa1a25c34e4172b27ff1ba30", "input_tokens": 347, "server_ms": 36.25}, {"id": "wyrd-scene-fletcher/f-go-tavern/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.05195500841364, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.78}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.310486011672765, "probs": [0.09180558671996532, 0.02967758775648927, 0.8785168255235455], "top": "tavern", "option_logits": null, "prompt_sha256": "2184bf6bdaf9684fb7d95a5bc48a0892df975609f94a3ed7848054c957e74993", "input_tokens": 400, "server_ms": 36.2}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.40377498557791, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.99}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.709282994735986, "probs": [0.13214384255458153, 0.8678561574454184], "top": "elsewhere", "option_logits": null, "prompt_sha256": "ddaba14f654fa4c16c488632b8f44e96e68d760a8f2bc42dec58235647f892ea", "input_tokens": 360, "server_ms": 35.97}, {"id": "wyrd-scene-fletcher/f-go-tavern/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.372183989267796, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.93}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.652006016112864, "probs": [0.31207496905784127, 0.025374130011708906, 0.6625509009304498], "top": "tavern", "option_logits": null, "prompt_sha256": "f79cd25f72d3225d918207c80cceb334da79a685d99944f8af093343a71507ac", "input_tokens": 412, "server_ms": 36.32}, {"id": "wyrd-scene-fletcher/f-go-tavern/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-go-tavern", "gold": ["tavern"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.3603589865379, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.41}, {"id": "wyrd-scene-fletcher/f-approach-wren/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.95752104232088, "probs": [0.10012611628383591, 0.8998738837161642], "top": "holds", "option_logits": null, "prompt_sha256": "8b0960d187761f892193b961e182db548cb51b368cc86fa64963b86ca31104dc", "input_tokens": 350, "server_ms": 36.01}, {"id": "wyrd-scene-fletcher/f-approach-wren/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.29676397377625, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.07}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.58919297577813, "probs": [0.08392095203238217, 0.910426262057999, 0.005652785909618714], "top": "meet_wren", "option_logits": null, "prompt_sha256": "8628e07529e354a0ea6b36af3d7c80bbb43b24fdb3f2d998c0aff54d2ff1eb74", "input_tokens": 403, "server_ms": 36.11}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.69334197510034, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.1}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.24925697641447, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "1c0c0864831cdb932a28d47ef8c6c43d7b1bbec5c04d6ef50e9bca20f3fc7656", "input_tokens": 363, "server_ms": 35.94}, {"id": "wyrd-scene-fletcher/f-approach-wren/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.042979969177395, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.71}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.22900102753192, "probs": [0.3609531260448983, 0.6348499071093584, 0.004196966845743212], "top": "meet_wren", "option_logits": null, "prompt_sha256": "2b709da6d4717d3d93eeea477d3c66ec4adb06c92a6f7caf53fef28d2529a32e", "input_tokens": 415, "server_ms": 36.39}, {"id": "wyrd-scene-fletcher/f-approach-wren/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-fletcher/f-approach-wren", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.596074983943254, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.17}, {"id": "wyrd-scene-fletcher/f-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.92060596588999, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "642f6317a50fce2972b9d827cd804fd97828010dda845423c67ffcb208e9b040", "input_tokens": 351, "server_ms": 36.21}, {"id": "wyrd-scene-fletcher/f-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.43405803432688, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.94}, {"id": "wyrd-scene-fletcher/f-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.49932103510946, "probs": [0.5349839022743945, 0.44320089104384086, 0.021815206681764517], "top": "stay", "option_logits": null, "prompt_sha256": "614a7bfac1376f57a1029284d6165b36ef01754e5c90758e79dade1a63f15749", "input_tokens": 404, "server_ms": 36.28}, {"id": "wyrd-scene-fletcher/f-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.69360798411071, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.98}, {"id": "wyrd-scene-fletcher/f-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.97572799120098, "probs": [0.9773408097411098, 0.022659190258890113], "top": "same", "option_logits": null, "prompt_sha256": "72422c0e5debc256907b288457c1797300cf448d5ec856a6f2e35c6decd9edf5", "input_tokens": 364, "server_ms": 36.35}, {"id": "wyrd-scene-fletcher/f-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.23816399322823, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.9}, {"id": "wyrd-scene-fletcher/f-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.704995010979474, "probs": [0.9484375641457526, 0.04668393422657173, 0.004878501627675742], "top": "stay", "option_logits": null, "prompt_sha256": "b0ed9a6aa86ee72d1817b29132d7507478803a28b48a926b991eeb24cd2f9e12", "input_tokens": 416, "server_ms": 36.45}, {"id": "wyrd-scene-fletcher/f-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.46440797438845, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.1}, {"id": "wyrd-scene-fletcher/f-leave-town/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.59253099979833, "probs": [0.7557061813028185, 0.24429381869718142], "top": "moved", "option_logits": null, "prompt_sha256": "bd8db9693ef78eb0beecc5c18fa010d5d06e6759044ecfab3d74ba1a865797de", "input_tokens": 345, "server_ms": 36.3}, {"id": "wyrd-scene-fletcher/f-leave-town/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.12888801563531, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.97}, {"id": "wyrd-scene-fletcher/f-leave-town/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.41708100261167, "probs": [0.3622300087461422, 0.5277941644393258, 0.10997582681453204], "top": "meet_wren", "option_logits": null, "prompt_sha256": "a14d21ff8e50b52b1976269e33c341c5e16dd10d600a65f7920af31041facb6a", "input_tokens": 398, "server_ms": 36.22}, {"id": "wyrd-scene-fletcher/f-leave-town/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.760873979423195, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.89}, {"id": "wyrd-scene-fletcher/f-leave-town/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.01047901250422, "probs": [0.04691279649254219, 0.9530872035074577], "top": "elsewhere", "option_logits": null, "prompt_sha256": "06d1489204fd88302faf9ecfd135ae4479e2b28f2eb94389357672f822467e63", "input_tokens": 358, "server_ms": 36.06}, {"id": "wyrd-scene-fletcher/f-leave-town/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.209865036886185, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 35.16}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.01498898956925, "probs": [0.6394729578033058, 0.3414721146107966, 0.019054927585897746], "top": "stay", "option_logits": null, "prompt_sha256": "f13ca14ecf5c1323b82136fe289cd1167d2b3da2b09f6f1401926403643820c4", "input_tokens": 410, "server_ms": 36.32}, {"id": "wyrd-scene-fletcher/f-leave-town/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-leave-town", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.608949994668365, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.28}, {"id": "wyrd-scene-fletcher/f-nightfall/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.642565024551004, "probs": [0.06310763191560703, 0.936892368084393], "top": "holds", "option_logits": null, "prompt_sha256": "83f722d8b501914e1e0d49966226acdabbc5bfd24d7db5a3288d83c54e788906", "input_tokens": 334, "server_ms": 35.93}, {"id": "wyrd-scene-fletcher/f-nightfall/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.33944600028917, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.25}, {"id": "wyrd-scene-fletcher/f-nightfall/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.80513902148232, "probs": [0.8867723079125559, 0.09266828976827934, 0.020559402319164796], "top": "stay", "option_logits": null, "prompt_sha256": "40cf512c5046c6f14722b7dc272d6edf32d8b26be6c39c5ba91ae15234b60458", "input_tokens": 387, "server_ms": 36.16}, {"id": "wyrd-scene-fletcher/f-nightfall/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.775620996020734, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 35.08}, {"id": "wyrd-scene-fletcher/f-nightfall/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.223734012339264, "probs": [0.981155009643547, 0.018844990356453022], "top": "same", "option_logits": null, "prompt_sha256": "892d3122f8d9956056bf21f902d2d8bb500c8c6ae105bd5c0f51ee27ce4f3e67", "input_tokens": 347, "server_ms": 35.75}, {"id": "wyrd-scene-fletcher/f-nightfall/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.80480200983584, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.10495603084564, "probs": [0.9595292782348949, 0.03451305801549889, 0.005957663749606297], "top": "stay", "option_logits": null, "prompt_sha256": "c1b2b2f57285ac7ed8baff6dc7a583e23b79a69d6d56df16e052514f1f683d50", "input_tokens": 399, "server_ms": 36.32}, {"id": "wyrd-scene-fletcher/f-nightfall/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-nightfall", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.180641018319875, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.25}, {"id": "wyrd-scene-fletcher/f-wren-comes/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.6680209916085, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fd10140bbf230f16298039677ff7eca598682b6b00c499cb87d8c4bc3636e84f", "input_tokens": 354, "server_ms": 35.91}, {"id": "wyrd-scene-fletcher/f-wren-comes/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.76050901738927, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.96}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.50251902220771, "probs": [0.14608219944812195, 0.8462617876902637, 0.007656012861614229], "top": "meet_wren", "option_logits": null, "prompt_sha256": "d6285e0f761cb72916991b3aa75e90969fd5d48162a2ab8547c25a134ac9970f", "input_tokens": 407, "server_ms": 36.26}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.31466096686199, "probs": [0.861023632332021, 0.08450597252838886, 0.05447039513959021], "top": "stay", "option_logits": null, "prompt_sha256": "413feb1627136dce565aac3bf50b3ea3808722e8a364b529517665b09fee4f2d", "input_tokens": 256, "server_ms": 34.97}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.00926002766937, "probs": [0.9652800911770798, 0.034719908822920295], "top": "same", "option_logits": null, "prompt_sha256": "16c77bb7f415e403a6f0e5f0fddc8a7efe2eff38eb8b950ba7a89c6b3a8f6089", "input_tokens": 367, "server_ms": 36.01}, {"id": "wyrd-scene-fletcher/f-wren-comes/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.44022299302742, "probs": [0.8815938286018441, 0.11840617139815589], "top": "same", "option_logits": null, "prompt_sha256": "925913696599b0023875e54a4b94ae84b72c6fbb2579330e215504f654a83f9d", "input_tokens": 216, "server_ms": 34.71}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.693967977073044, "probs": [0.7741398924975833, 0.22074230029695507, 0.0051178072054615525], "top": "stay", "option_logits": null, "prompt_sha256": "3635b3c92721ae42861023028c1b225f3121b020a74e9a609e040b88650d7855", "input_tokens": 419, "server_ms": 36.36}, {"id": "wyrd-scene-fletcher/f-wren-comes/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-fletcher/f-wren-comes", "gold": ["meet_wren"], "forbid": [], "option_ids": ["stay", "meet_wren", "tavern"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.07886202353984, "probs": [0.9296767595752046, 0.0588135569838071, 0.011509683440988398], "top": "stay", "option_logits": null, "prompt_sha256": "546a97583d85218c1fb4f2a61eb5dc5e9493c64137139a6fe01cb443b663e73c", "input_tokens": 268, "server_ms": 35.13}, {"id": "wyrd-scene-toll/t-reads-letter/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.123325971886516, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "f0ffc47c17537a4d4fb3fb4b8ac9997fa8107f55e51e4ad0b9a13866601ee85a", "input_tokens": 336, "server_ms": 36.06}, {"id": "wyrd-scene-toll/t-reads-letter/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.95211704308167, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.95}, {"id": "wyrd-scene-toll/t-reads-letter/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 60.191980039235204, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "471aea5565661294ab762c328e9ed37c4e9cc9768cbace3f72555836d5d0980f", "input_tokens": 370, "server_ms": 40.58}, {"id": "wyrd-scene-toll/t-reads-letter/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.06107101589441, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 35.33}, {"id": "wyrd-scene-toll/t-reads-letter/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.45013596909121, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8ff0421aea2c2f4766e31e994012e906a77b3d0ee479e0b5c37a05284a6df18b", "input_tokens": 349, "server_ms": 35.57}, {"id": "wyrd-scene-toll/t-reads-letter/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.03050403017551, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.83}, {"id": "wyrd-scene-toll/t-reads-letter/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.04117301804945, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "1da50f17674a5941afd0eb62df0f2542809450c13dc4e4c87ac21ad3c668d9a0", "input_tokens": 381, "server_ms": 35.79}, {"id": "wyrd-scene-toll/t-reads-letter/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-reads-letter", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.32911899313331, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.73}, {"id": "wyrd-scene-toll/t-questions/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.57106198789552, "probs": [0.07519394939170732, 0.9248060506082927], "top": "holds", "option_logits": null, "prompt_sha256": "8da13952e1ce2f0ba71508070020293f826b683c09307d4ce80e0f9861a43742", "input_tokens": 344, "server_ms": 36.06}, {"id": "wyrd-scene-toll/t-questions/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.1910210265778, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.16}, {"id": "wyrd-scene-toll/t-questions/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.22475095838308, "probs": [0.5930110454411601, 0.4069889545588398], "top": "stay", "option_logits": null, "prompt_sha256": "579b6c112541b6cf0c35d7dae107c74aaec53a1c6ccde6fe9114f85865da6bcb", "input_tokens": 378, "server_ms": 35.96}, {"id": "wyrd-scene-toll/t-questions/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.97553999116644, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.96}, {"id": "wyrd-scene-toll/t-questions/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.22361200908199, "probs": [0.9608203271108413, 0.03917967288915878], "top": "same", "option_logits": null, "prompt_sha256": "5711e6ad3ff00a7cdddb958f6145cb154b6347e0ced0fd5eb08ce372d6d497b2", "input_tokens": 357, "server_ms": 36.28}, {"id": "wyrd-scene-toll/t-questions/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.533752041403204, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.87}, {"id": "wyrd-scene-toll/t-questions/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.16102699423209, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "6132ad59e49d1e8272bf1ee3140ebd2c25d41f4c0534a3b4ea364850bb197d54", "input_tokens": 389, "server_ms": 36.16}, {"id": "wyrd-scene-toll/t-questions/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-questions", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.06667304830626, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.91}, {"id": "wyrd-scene-toll/t-future/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.64460700750351, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "fa790ec1111537a51cf632a4886f04447309aa0b5e7c0dfa6b5a9a07c1e819a8", "input_tokens": 340, "server_ms": 36.13}, {"id": "wyrd-scene-toll/t-future/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.25264502223581, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.9}, {"id": "wyrd-scene-toll/t-future/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.14919601753354, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "0de1259ac2b6b448a46489f29b01b2670fccfaca9910607d4810e3968f5baef5", "input_tokens": 374, "server_ms": 36.07}, {"id": "wyrd-scene-toll/t-future/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.030285984277725, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 35.23}, {"id": "wyrd-scene-toll/t-future/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.56462496658787, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "29b4f0fa0f41196d840255921fdb8fcc99c92dc315f62a567fb57f9e15fe06b2", "input_tokens": 353, "server_ms": 35.93}, {"id": "wyrd-scene-toll/t-future/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.734546014107764, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.89}, {"id": "wyrd-scene-toll/t-future/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.87564200302586, "probs": [0.9583890938912367, 0.04161090610876325], "top": "stay", "option_logits": null, "prompt_sha256": "9462f939e7a16c19c8b8b6fba9b27b798f9065a36017e89060953bf0e6681042", "input_tokens": 385, "server_ms": 36.16}, {"id": "wyrd-scene-toll/t-future/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-future", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.703218000009656, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.97}, {"id": "wyrd-scene-toll/t-granted/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.9470849907957, "probs": [0.5, 0.5], "top": "moved", "option_logits": null, "prompt_sha256": "104779576777ff5bf41044991788adda2b927d0469b6d69428c3706953bbbdd6", "input_tokens": 333, "server_ms": 36.36}, {"id": "wyrd-scene-toll/t-granted/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.29522804822773, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.94}, {"id": "wyrd-scene-toll/t-granted/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.41479996079579, "probs": [0.07967467169628852, 0.9203253283037116], "top": "departure", "option_logits": null, "prompt_sha256": "80d7c857a021182a0e28410610c85e4bb689ad32ec1a49db5b03488bfcaa7c72", "input_tokens": 367, "server_ms": 35.93}, {"id": "wyrd-scene-toll/t-granted/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.38538101548329, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.65}, {"id": "wyrd-scene-toll/t-granted/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.83209399972111, "probs": [0.09461348765523125, 0.9053865123447689], "top": "elsewhere", "option_logits": null, "prompt_sha256": "b5fbb833a56aefeaf1de894421c5e319efaca37f356092e7a5c357753263180a", "input_tokens": 346, "server_ms": 36.05}, {"id": "wyrd-scene-toll/t-granted/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.31074203038588, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.79}, {"id": "wyrd-scene-toll/t-granted/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.370725993067026, "probs": [0.11840617139815589, 0.8815938286018441], "top": "departure", "option_logits": null, "prompt_sha256": "889f6a51b6a2bd61c23f0c6cbfed81fec9447035b045fffd8796cf63ee5def92", "input_tokens": 378, "server_ms": 36.06}, {"id": "wyrd-scene-toll/t-granted/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-toll/t-granted", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.791894995607436, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.88}, {"id": "wyrd-scene-toll/t-denied/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.69701997935772, "probs": [0.2681939143399331, 0.7318060856600669], "top": "holds", "option_logits": null, "prompt_sha256": "46f9344d4111977545d8091956ab9efe9425f4d6511ba9b443cced3522ded682", "input_tokens": 339, "server_ms": 36.02}, {"id": "wyrd-scene-toll/t-denied/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.86619899561629, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.03}, {"id": "wyrd-scene-toll/t-denied/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.27099100779742, "probs": [0.059498364613396776, 0.9405016353866033], "top": "departure", "option_logits": null, "prompt_sha256": "b5e27df6fbeca3dd7f99845662646461bb8ad246781d2f8ad0712f1cf43ef54f", "input_tokens": 373, "server_ms": 36.0}, {"id": "wyrd-scene-toll/t-denied/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.592397016473114, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.82}, {"id": "wyrd-scene-toll/t-denied/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.23119501210749, "probs": [0.08439803417650704, 0.9156019658234928], "top": "elsewhere", "option_logits": null, "prompt_sha256": "a7e8b5bc0fe90e112eb98d9dafa54c20c9aecbcefcb6b8e4050cd3ed195cdc19", "input_tokens": 352, "server_ms": 35.96}, {"id": "wyrd-scene-toll/t-denied/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.08282398618758, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.93}, {"id": "wyrd-scene-toll/t-denied/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.70468004932627, "probs": [0.1636682776269396, 0.8363317223730603], "top": "departure", "option_logits": null, "prompt_sha256": "1044beed5a145c3915e5d5944e400f80588b457fac8359c8a84c817fb1d4c5bd", "input_tokens": 384, "server_ms": 36.04}, {"id": "wyrd-scene-toll/t-denied/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-denied", "gold": ["departure"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.147690021432936, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.94}, {"id": "wyrd-scene-toll/t-rider/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.50469901552424, "probs": [0.08439803417650704, 0.9156019658234928], "top": "holds", "option_logits": null, "prompt_sha256": "6d774ece16034c26659f29265befd7da3d8b79628b85df58964648caf5189541", "input_tokens": 337, "server_ms": 36.14}, {"id": "wyrd-scene-toll/t-rider/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.25369199272245, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.16}, {"id": "wyrd-scene-toll/t-rider/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.38885395415127, "probs": [0.7439389554145914, 0.25606104458540857], "top": "stay", "option_logits": null, "prompt_sha256": "ad3ed2c45db9ce5011d3d2370be2f3d968680c7f18cca8fd002a9b15e518f42c", "input_tokens": 371, "server_ms": 36.02}, {"id": "wyrd-scene-toll/t-rider/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.9326110156253, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 35.07}, {"id": "wyrd-scene-toll/t-rider/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.53802198078483, "probs": [0.9727760393154234, 0.027223960684576436], "top": "same", "option_logits": null, "prompt_sha256": "0a21577e5a3493e40ec4582380c3052673ed73aa5fb60625fac3cf421bec7d5b", "input_tokens": 350, "server_ms": 35.85}, {"id": "wyrd-scene-toll/t-rider/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.89695998746902, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 35.07}, {"id": "wyrd-scene-toll/t-rider/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.73143403651193, "probs": [0.9558139322179339, 0.04418606778206604], "top": "stay", "option_logits": null, "prompt_sha256": "f136eb34df26d51faa1026c993cd7c14f3acf80651e63a25018b1f2fd861f508", "input_tokens": 382, "server_ms": 36.1}, {"id": "wyrd-scene-toll/t-rider/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-rider", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.20522200781852, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.99}, {"id": "wyrd-scene-toll/t-city-memory/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.537194035016, "probs": [0.09461348765523125, 0.9053865123447689], "top": "holds", "option_logits": null, "prompt_sha256": "2a84bb9ff897a68b4da6023adb34a6dd5c395cc9586d40ec85e756627a9c79f7", "input_tokens": 343, "server_ms": 36.1}, {"id": "wyrd-scene-toll/t-city-memory/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.59439901076257, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.03}, {"id": "wyrd-scene-toll/t-city-memory/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.94173098122701, "probs": [0.6932998143038492, 0.30670018569615076], "top": "stay", "option_logits": null, "prompt_sha256": "8c917d9beba3c3c7a7a6f9bdb2ee69fd08c2bb3224798b4855e51f8f0a90765a", "input_tokens": 377, "server_ms": 36.11}, {"id": "wyrd-scene-toll/t-city-memory/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.96935100341216, "probs": [0.8275619327448428, 0.1724380672551571], "top": "stay", "option_logits": null, "prompt_sha256": "8478d6de3a8afe4e8368f676fd79368c05b3dbb00f81f4944c24eb2c9499ad47", "input_tokens": 237, "server_ms": 34.91}, {"id": "wyrd-scene-toll/t-city-memory/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.400728025939316, "probs": [0.9710643566281367, 0.02893564337186314], "top": "same", "option_logits": null, "prompt_sha256": "9c6b567db8d46f95e55546abe5bf9bb2fb8d6bb56bde98e9c930c4e98c75410c", "input_tokens": 356, "server_ms": 36.08}, {"id": "wyrd-scene-toll/t-city-memory/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.546217968687415, "probs": [0.8678561574454184, 0.13214384255458153], "top": "same", "option_logits": null, "prompt_sha256": "08fc191a1243b19f9b127f3f603e4ee4a15d0557a5a8883fa111a1ddce2023ce", "input_tokens": 216, "server_ms": 34.85}, {"id": "wyrd-scene-toll/t-city-memory/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.101369973272085, "probs": [0.9471470189565372, 0.052852981043462664], "top": "stay", "option_logits": null, "prompt_sha256": "31cd8b9fda0335aa6dc3316f9b7cf36420717a3bd2dab6a48387f215bff22738", "input_tokens": 388, "server_ms": 36.41}, {"id": "wyrd-scene-toll/t-city-memory/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-toll/t-city-memory", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "departure"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.91478800447658, "probs": [0.9330797425441405, 0.06692025745585947], "top": "stay", "option_logits": null, "prompt_sha256": "ef5595471954fe6548e11c82d8a13dfec0fa50e37cbc02e68dadbd30e3f37458", "input_tokens": 248, "server_ms": 34.93}, {"id": "wyrd-scene-vigil/v-talk/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.305970028508455, "probs": [0.07967467169628852, 0.9203253283037116], "top": "holds", "option_logits": null, "prompt_sha256": "40f2adc7989c6b14b17ba0c1d6590818f12f396ab6cbfc46ae06729ec5bee8f5", "input_tokens": 352, "server_ms": 36.23}, {"id": "wyrd-scene-vigil/v-talk/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.526189003605396, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 34.96}, {"id": "wyrd-scene-vigil/v-talk/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.70536695793271, "probs": [0.8940776460411956, 0.10592235395880453], "top": "stay", "option_logits": null, "prompt_sha256": "19925a416cc32c92d0efe4692be8e6631ced06fbe97257f5fb5c6fdc67772001", "input_tokens": 382, "server_ms": 35.93}, {"id": "wyrd-scene-vigil/v-talk/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.69537801016122, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.74}, {"id": "wyrd-scene-vigil/v-talk/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.43685699719936, "probs": [0.9743891388556707, 0.0256108611443294], "top": "same", "option_logits": null, "prompt_sha256": "8723c322393045642eddb7ef97d3fecb07bef0deb1f322fa3332b4b30cf28871", "input_tokens": 362, "server_ms": 36.02}, {"id": "wyrd-scene-vigil/v-talk/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.26706200139597, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.88}, {"id": "wyrd-scene-vigil/v-talk/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.66078798053786, "probs": [0.9759090252955993, 0.024090974704400647], "top": "stay", "option_logits": null, "prompt_sha256": "a2b5215073a29bb035358ae2287afedfb46f5e708804175670059d551ea6de6c", "input_tokens": 394, "server_ms": 36.09}, {"id": "wyrd-scene-vigil/v-talk/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-talk", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.350162983406335, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.77}, {"id": "wyrd-scene-vigil/v-step-out/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.15266996715218, "probs": [0.8678561574454184, 0.13214384255458153], "top": "moved", "option_logits": null, "prompt_sha256": "e489f9a1cd920b56d4ae854926008262e016de255c4adb8484814fe32a9e8882", "input_tokens": 348, "server_ms": 36.0}, {"id": "wyrd-scene-vigil/v-step-out/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.90673999115825, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.03}, {"id": "wyrd-scene-vigil/v-step-out/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.80066699581221, "probs": [0.04691279649254219, 0.9530872035074577], "top": "outside", "option_logits": null, "prompt_sha256": "0ffc0574cb969677d620e187a81adb37b7d30aa874903503f9dd1e07837af339", "input_tokens": 378, "server_ms": 35.83}, {"id": "wyrd-scene-vigil/v-step-out/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.58680902281776, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.24}, {"id": "wyrd-scene-vigil/v-step-out/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.36582799302414, "probs": [0.07094578364611796, 0.9290542163538821], "top": "elsewhere", "option_logits": null, "prompt_sha256": "8a3b3b00b6db09e9a1c0c6661a30c4d36463f3a3fd67de40cf7bb8c52708b58f", "input_tokens": 358, "server_ms": 35.95}, {"id": "wyrd-scene-vigil/v-step-out/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.98694903124124, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.59}, {"id": "wyrd-scene-vigil/v-step-out/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.560545017011464, "probs": [0.11840617139815589, 0.8815938286018441], "top": "outside", "option_logits": null, "prompt_sha256": "d292a9d85016c07432556d4f8c64c353f0c4c6e30fb40c02a64bd7de3785cb92", "input_tokens": 390, "server_ms": 36.4}, {"id": "wyrd-scene-vigil/v-step-out/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "control", "group": "wyrd-scene-vigil/v-step-out", "gold": ["outside"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.50935098249465, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.21}, {"id": "wyrd-scene-vigil/v-plan-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.951201018411666, "probs": [0.08937422170979005, 0.91062577829021], "top": "holds", "option_logits": null, "prompt_sha256": "83812b5995e356edb72fb63f547426451f13d5aa6daf3781acdc918b3970cdc7", "input_tokens": 354, "server_ms": 36.03}, {"id": "wyrd-scene-vigil/v-plan-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.50931000430137, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.32}, {"id": "wyrd-scene-vigil/v-plan-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.49226900236681, "probs": [0.767101934678807, 0.23289806532119295], "top": "stay", "option_logits": null, "prompt_sha256": "7ec259f1effd8d126c11d95e152cfc69942a6fc431dad59cecccc25f56168f9e", "input_tokens": 384, "server_ms": 36.41}, {"id": "wyrd-scene-vigil/v-plan-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.414116031490266, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.08}, {"id": "wyrd-scene-vigil/v-plan-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.88345901155844, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "618b0cbba4c5132896d2aee5b51a23006cbb2e91c9c11e1cfa286243c23e78ab", "input_tokens": 364, "server_ms": 35.93}, {"id": "wyrd-scene-vigil/v-plan-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.12391399638727, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.77}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.39713102532551, "probs": [0.9652800911770798, 0.034719908822920295], "top": "stay", "option_logits": null, "prompt_sha256": "d50e46ef0be9a2b8c3c650006f81ea521b15d7d9047b323eb4048093ab8a4684", "input_tokens": 396, "server_ms": 36.37}, {"id": "wyrd-scene-vigil/v-plan-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-plan-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.004210989456624, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.37}, {"id": "wyrd-scene-vigil/v-to-pub/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.42595505155623, "probs": [0.767101934678807, 0.23289806532119295], "top": "moved", "option_logits": null, "prompt_sha256": "33ff3d414bc5f8d80926944a87753e3b98cfb7ef95e870a766e9afe95eebfe6b", "input_tokens": 357, "server_ms": 36.23}, {"id": "wyrd-scene-vigil/v-to-pub/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["moved"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.224703995510936, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.27}, {"id": "wyrd-scene-vigil/v-to-pub/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.621173010673374, "probs": [0.23289806532119295, 0.767101934678807], "top": "outside", "option_logits": null, "prompt_sha256": "47f570fe7c70437d9840c3fd2f91c462a3747f17c442fca741bbea6367cf0142", "input_tokens": 387, "server_ms": 36.12}, {"id": "wyrd-scene-vigil/v-to-pub/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.421073011122644, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 35.04}, {"id": "wyrd-scene-vigil/v-to-pub/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.9978599883616, "probs": [0.04979902705524531, 0.9502009729447547], "top": "elsewhere", "option_logits": null, "prompt_sha256": "94600a9c2b14d68dbeda521a432fd8af6671323581dee61a746c494809960e8e", "input_tokens": 367, "server_ms": 36.36}, {"id": "wyrd-scene-vigil/v-to-pub/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["elsewhere"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.30365902045742, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.95}, {"id": "wyrd-scene-vigil/v-to-pub/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.6628149934113, "probs": [0.7064759712765732, 0.2935240287234267], "top": "stay", "option_logits": null, "prompt_sha256": "a5113e5b18a3441af4dee285c7d72071b15245573a67427c5221de929c83685d", "input_tokens": 399, "server_ms": 36.33}, {"id": "wyrd-scene-vigil/v-to-pub/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-to-pub", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.5533659751527, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.48}, {"id": "wyrd-scene-vigil/v-vision/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 56.24806403648108, "probs": [0.11840617139815589, 0.8815938286018441], "top": "holds", "option_logits": null, "prompt_sha256": "b17ebebc50c5bce85b78dcd0cdcc7e28d1a690059d2b94b0d881749ac0c2434d", "input_tokens": 356, "server_ms": 36.43}, {"id": "wyrd-scene-vigil/v-vision/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.97136794636026, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.05}, {"id": "wyrd-scene-vigil/v-vision/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.65673603955656, "probs": [0.8998738837161642, 0.10012611628383591], "top": "stay", "option_logits": null, "prompt_sha256": "fd039242cd0d6744d746b04a6ef642c34b39277d28dd1d4bf60d2a5e8cf3d5c0", "input_tokens": 386, "server_ms": 36.22}, {"id": "wyrd-scene-vigil/v-vision/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 52.959508961066604, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.97}, {"id": "wyrd-scene-vigil/v-vision/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.99888502527028, "probs": [0.967322429380235, 0.03267757061976492], "top": "same", "option_logits": null, "prompt_sha256": "ba649355aa2ef18cd1db8c4e403d7c809ed5b0563e2dfa4e0ee4082a79501382", "input_tokens": 366, "server_ms": 36.26}, {"id": "wyrd-scene-vigil/v-vision/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.99964901152998, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 35.1}, {"id": "wyrd-scene-vigil/v-vision/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.351064016576856, "probs": [0.9710643566281367, 0.02893564337186314], "top": "stay", "option_logits": null, "prompt_sha256": "4685b10393950098c6746d910e3c07b5a2fae5bd3072d6ac3f4d485a72535e4e", "input_tokens": 398, "server_ms": 36.48}, {"id": "wyrd-scene-vigil/v-vision/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-vision", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.38968400610611, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 34.94}, {"id": "wyrd-scene-vigil/v-demon-enters/place", "set": "wyrd", "cond": "evidence", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.97707496397197, "probs": [0.11201237577359563, 0.8879876242264044], "top": "holds", "option_logits": null, "prompt_sha256": "3e60dece66b3ae937d8fad2df6f6a8236bc5fae4da2969b513d14780916b1ce8", "input_tokens": 344, "server_ms": 36.2}, {"id": "wyrd-scene-vigil/v-demon-enters/place#blind", "set": "wyrd", "cond": "blind", "decision": "place", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["holds"], "forbid": [], "option_ids": ["moved", "holds"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.51271898346022, "probs": [0.1395062328236417, 0.8604937671763584], "top": "holds", "option_logits": null, "prompt_sha256": "cb95dbf02297ce387370428751bffbaa1f6441fc19b0abd6c6f657b74ddc9ee4", "input_tokens": 203, "server_ms": 35.32}, {"id": "wyrd-scene-vigil/v-demon-enters/exit", "set": "wyrd", "cond": "evidence", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.6166139645502, "probs": [0.8678561574454184, 0.13214384255458153], "top": "stay", "option_logits": null, "prompt_sha256": "0a742c9ee056696b7c854643676a7ea0543ca2db31e9535f048a3b5749f0c275", "input_tokens": 374, "server_ms": 35.9}, {"id": "wyrd-scene-vigil/v-demon-enters/exit#blind", "set": "wyrd", "cond": "blind", "decision": "exit", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.39056299999356, "probs": [0.8815938286018441, 0.11840617139815589], "top": "stay", "option_logits": null, "prompt_sha256": "c99038ee9287ceb8bc8afec9f05a8ecfa6fff4f5b61b0cf61490b3f2260a4820", "input_tokens": 233, "server_ms": 34.74}, {"id": "wyrd-scene-vigil/v-demon-enters/place2", "set": "wyrd", "cond": "evidence", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.03052001586184, "probs": [0.9583890938912367, 0.04161090610876325], "top": "same", "option_logits": null, "prompt_sha256": "ff67de69071213b7539981b64f0c2e1c4eb33584039ae35b76b414ef384f9f02", "input_tokens": 354, "server_ms": 35.88}, {"id": "wyrd-scene-vigil/v-demon-enters/place2#blind", "set": "wyrd", "cond": "blind", "decision": "place2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["same"], "forbid": [], "option_ids": ["same", "elsewhere"], "gold_desc_id": null, "ok": true, "e2e_ms": 53.20089199813083, "probs": [0.9053865123447689, 0.09461348765523125], "top": "same", "option_logits": null, "prompt_sha256": "d5ba767952be30fc3c6752b330ac3949468a53c902b99ee1a3321c310325c2bc", "input_tokens": 213, "server_ms": 34.75}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2", "set": "wyrd", "cond": "evidence", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 54.20566600514576, "probs": [0.9692484548756632, 0.03075154512433679], "top": "stay", "option_logits": null, "prompt_sha256": "8a7f414fb4be8133416dedd3a92c09511a2a35f686a9576a0245a8b4ea9aae99", "input_tokens": 386, "server_ms": 35.89}, {"id": "wyrd-scene-vigil/v-demon-enters/exit2#blind", "set": "wyrd", "cond": "blind", "decision": "exit2", "tag": "case", "group": "wyrd-scene-vigil/v-demon-enters", "gold": ["stay"], "forbid": [], "option_ids": ["stay", "outside"], "gold_desc_id": null, "ok": true, "e2e_ms": 55.58886198559776, "probs": [0.8879876242264044, 0.11201237577359563], "top": "stay", "option_logits": null, "prompt_sha256": "4b9dc02123a54f21541de3b9d814c83c6eaaa43fb3426bfa3bb3d7001e702abf", "input_tokens": 245, "server_ms": 35.0}], "seconds": 61.7, "failures": 0}, "negative": {"rows": [{"id": "a3f18f3a63d45345942b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.400469991378486, "probs": [0.06566437538338366, 0.8598927385077062, 0.07444288610891006], "top": "insufficient", "option_logits": null, "prompt_sha256": "2111f6636d10660f740e0f032908924d0640e60a712060af19c6e8972d95e5c9", "input_tokens": 204, "server_ms": 35.44}, {"id": "f40beba9088c8db8bbd6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.53602400282398, "probs": [0.2945662267563794, 0.03956295740237862, 0.665870815841242], "top": "contradicted", "option_logits": null, "prompt_sha256": "ed5aa8d62ce8778e40b8c51f1d1fe90b1f392ab252247908adc4e84b2546ab08", "input_tokens": 202, "server_ms": 35.29}, {"id": "0b43ea8e24e74d621f1b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.615917975548655, "probs": [0.9241406947470572, 0.051569175242271295, 0.024290130010671554], "top": "insufficient", "option_logits": null, "prompt_sha256": "f6f696700120ee981b3e3e0dc237f67896db84867803273e124be96bfd72fecc", "input_tokens": 196, "server_ms": 35.42}, {"id": "942ec65dac0bfb9874f9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2fdaa8a61e6e989dd866", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.585310000926256, "probs": [0.11215130215053613, 0.052825544241278395, 0.8350231536081855], "top": "contradicted", "option_logits": null, "prompt_sha256": "ef4b71a8ae0a1bc3711450f77606ddabd2be62c9c2b6587e1fcacbccfcb9cb00", "input_tokens": 196, "server_ms": 35.26}, {"id": "e0c140e9222d1bce5647", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.82413201732561, "probs": [0.15307015041158983, 0.014109646797217717, 0.8328202027911924], "top": "insufficient", "option_logits": null, "prompt_sha256": "215840810bc2483aa303cd79f5d4f92093a49aa9957dfa697f20989aa1390d55", "input_tokens": 209, "server_ms": 35.37}, {"id": "8e8c3804a3c15ebb31e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.799919958692044, "probs": [0.03329916574926079, 0.8166100641505288, 0.1500907701002104], "top": "supported", "option_logits": null, "prompt_sha256": "fc0d6607f95e1b53fe2cfa889a2cf1bfed2a95e2da0424c85742050fd8b5cde7", "input_tokens": 206, "server_ms": 34.95}, {"id": "56c419b12cd37cbe790c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.57555899536237, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "supported", "option_logits": null, "prompt_sha256": "f3bf4ddaa8f4178e0b5f59c09fffe0c07bfd4bd34d616bbf9577361257ca1afa", "input_tokens": 208, "server_ms": 35.25}, {"id": "75b470586834df8c568d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f4bea3d3bcc01056740d", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.06448699068278, "probs": [0.3541639151292647, 0.5160413123689802, 0.1297947725017551], "top": "contradicted", "option_logits": null, "prompt_sha256": "e5578ff7fa77b4e9f8593ede9b8e31ecccdca126f02444f3c94cdd0bb01ee6ca", "input_tokens": 200, "server_ms": 35.2}, {"id": "834f3b3e10cc61b33d25", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 55.004118010401726, "probs": [0.017835936327419456, 0.8721495360638406, 0.11001452760873995], "top": "insufficient", "option_logits": null, "prompt_sha256": "4b2ae5222808313b49d4b84a395fe3c5b48345949fa231e2c8fba100ed470a94", "input_tokens": 194, "server_ms": 35.46}, {"id": "76c7a79b1ff2363a60c5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.78351196506992, "probs": [0.045766307897467155, 0.024438745902358346, 0.9297949462001746], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b42ed76cec9fa4a467694c0069d312883b296f2ee2385059235881dee283be", "input_tokens": 194, "server_ms": 35.08}, {"id": "36c71bb649b7d5eccb1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.74246200174093, "probs": [0.03186734083364, 0.0247947114952375, 0.9433379476711226], "top": "contradicted", "option_logits": null, "prompt_sha256": "d2c60f4c3e6615ad87fdb4cd474622ea0a7f3f3b3fd7df582b5539d02a9dbd1b", "input_tokens": 193, "server_ms": 35.45}, {"id": "eac5121a4a1bf391a08c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "b51b300da2936decc6b7", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.48870296822861, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "contradicted", "option_logits": null, "prompt_sha256": "89f547d1f4dbfccb3c2e3443f18b4576344bd16efeaec8108d2febb0c913ec9e", "input_tokens": 192, "server_ms": 35.13}, {"id": "44574c44e243ee1b2f52", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.36300300061703, "probs": [0.03540454160362492, 0.924457767599498, 0.04013769079687701], "top": "contradicted", "option_logits": null, "prompt_sha256": "6beeb2f79fa6330dc68f2b769cf2779997d4b57c459bf40113bbc11a80353072", "input_tokens": 205, "server_ms": 35.4}, {"id": "2608370d01783735c3ea", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.09720801981166, "probs": [0.028539740573724572, 0.7452090392508762, 0.22625122017539928], "top": "insufficient", "option_logits": null, "prompt_sha256": "b094a32aab8f6f92e09b2405572effe6bec478d6d3b5985cdeb71dab794ed591", "input_tokens": 205, "server_ms": 35.1}, {"id": "ec94ef909f7918695ab3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 55.60622696066275, "probs": [0.026796810283021767, 0.844600378585805, 0.12860281113117333], "top": "insufficient", "option_logits": null, "prompt_sha256": "4e199755434441474e22e920ad5f8587827e30212d91945271ceadbb26337008", "input_tokens": 199, "server_ms": 35.74}, {"id": "03d919fd0e9b3a7cd1e6", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e4cfd694c80e08d3e6a5", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.6214750027284, "probs": [0.1303439116251751, 0.013621023271142042, 0.8560350651036829], "top": "contradicted", "option_logits": null, "prompt_sha256": "d75aa0e009db26e5e38e3bfd553573eccf95a0526360da23bada050f23e908f8", "input_tokens": 194, "server_ms": 35.14}, {"id": "86a15618ba4cb90a307e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 54.10011700587347, "probs": [0.1369403800499024, 0.018392355065480374, 0.8446672648846173], "top": "supported", "option_logits": null, "prompt_sha256": "9e994209df771b9f8829dd54c36907e41ed241a507cce230c7e3505302a9fa1e", "input_tokens": 198, "server_ms": 35.37}, {"id": "3dd9c3ac98f8ddf7bdd3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.461009985767305, "probs": [0.7651520472632017, 0.10276696393112311, 0.13208098880567515], "top": "contradicted", "option_logits": null, "prompt_sha256": "45785396c292fe5cbe62d99a45714d49c993d6f5b5783769cec1795f7b84f35f", "input_tokens": 198, "server_ms": 35.13}, {"id": "36273dd4777876a26338", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.247603005729616, "probs": [0.9249067812606918, 0.05495367443647756, 0.020139544302830666], "top": "contradicted", "option_logits": null, "prompt_sha256": "490eb806b0921b46aa816c18534ea40c0719e89eb76c34feb4e89d55c921bf36", "input_tokens": 194, "server_ms": 35.31}, {"id": "07ee70cfe4b01330ce89", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "20460d4ae0db46091e84", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.851181000936776, "probs": [0.0190993007707109, 0.7736996936098133, 0.2072010056194758], "top": "contradicted", "option_logits": null, "prompt_sha256": "7e29e327fc2aef321d690607a7c7347871c7eb3dd07ac6d45327aab094603d91", "input_tokens": 194, "server_ms": 35.23}, {"id": "51ceab3ce622c7c759f1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.38569200830534, "probs": [0.021206655711830842, 0.5200601388697108, 0.45873320541845836], "top": "insufficient", "option_logits": null, "prompt_sha256": "ffb27e8dfc1d8298a601b7f483c42104d68f6e8ebfea086896afd140756b8c6c", "input_tokens": 211, "server_ms": 35.1}, {"id": "b8091be929b1e489fcc4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.05045802379027, "probs": [0.9097361205627902, 0.05076536678270011, 0.039498512654509606], "top": "supported", "option_logits": null, "prompt_sha256": "e7855c00f037441fcdc5ff3fefaa61ddaeb0e79e09ba6c0c6043c767ee6d98ca", "input_tokens": 211, "server_ms": 34.94}, {"id": "c89b86e3f55123bfa506", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.487962984945625, "probs": [0.9223434223370909, 0.04004589025644397, 0.03761068740646525], "top": "supported", "option_logits": null, "prompt_sha256": "1a0c7ecb563009263a7969e56c6e00fc34f99b8ad08b4db579ed9e2bc78f4c6f", "input_tokens": 201, "server_ms": 35.2}, {"id": "3b0958cabc339005cf66", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "51b91ba31984c5939c1c", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.61881398130208, "probs": [0.1051237595843464, 0.007539451491827663, 0.8873367889238258], "top": "contradicted", "option_logits": null, "prompt_sha256": "610dae3d86ea66ae02b343c18c1478ef66be2619fe3a6c03ff05a434fac307d6", "input_tokens": 197, "server_ms": 35.24}, {"id": "f2b4ec4930322fe45118", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.99949401523918, "probs": [0.7409240241997488, 0.18635729477403776, 0.07271868102621346], "top": "insufficient", "option_logits": null, "prompt_sha256": "52c15cca48120ff357156a28827735982f0407b36b9806dc79a94193e52ed2a4", "input_tokens": 212, "server_ms": 35.11}, {"id": "61ac20b07a5189231e15", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.040571976453066, "probs": [0.9070425995126343, 0.07852475778895956, 0.01443264269840612], "top": "prohibited", "option_logits": null, "prompt_sha256": "5df6c8de75514db23d6e53210ba8920e11a83a08e31ffd76867eb6d789db3f29", "input_tokens": 218, "server_ms": 35.05}, {"id": "4b3d794c004566610181", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.58402500860393, "probs": [0.04968326826324694, 0.890344493113758, 0.05997223862299509], "top": "insufficient", "option_logits": null, "prompt_sha256": "3dc91e8f64358377c40956ef40870819732d6e66fc8ed44530ee8b7b1e32f1e6", "input_tokens": 203, "server_ms": 35.29}, {"id": "b7c4e62d07986286fce0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "770c1607f0a11f88f81e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.64206596277654, "probs": [0.09374299764974467, 0.009200493913293934, 0.8970565084369614], "top": "prohibited", "option_logits": null, "prompt_sha256": "866701419f4e54bef33f2151b0e4c0c830728dbfd94c6ba4a3fbd92b438151a2", "input_tokens": 209, "server_ms": 35.03}, {"id": "9473603b44af63beab79", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.996536997146904, "probs": [0.20575303419639995, 0.025954061735184695, 0.7682929040684153], "top": "prohibited", "option_logits": null, "prompt_sha256": "6c14eaaad2ab386d458a6458fce2b79bcd81ecbfc3c66679c950d808dd0c2a3f", "input_tokens": 209, "server_ms": 35.31}, {"id": "d41072dc35d8cd14c056", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 53.8687200169079, "probs": [0.8315971903507323, 0.10489918128594154, 0.06350362836332618], "top": "permitted", "option_logits": null, "prompt_sha256": "d8b705bc82e40643b8dcd135b43f67a8e3671ecc2f28e81b48de6aa1cbb84792", "input_tokens": 214, "server_ms": 35.08}, {"id": "33d6e5da58f0fee2490d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 55.26117602130398, "probs": [0.258643657502053, 0.6628301324407543, 0.07852621005719278], "top": "insufficient", "option_logits": null, "prompt_sha256": "5000e3b05641e28823047872c8bd19f4604d76e0fa38c23bdad3d8e7be7bce61", "input_tokens": 209, "server_ms": 35.28}, {"id": "823bf736b92e9d8fa66d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ffd081ada3bd9be9207a", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.247942007146776, "probs": [0.7942717885829101, 0.19977533322643162, 0.00595287819065825], "top": "prohibited", "option_logits": null, "prompt_sha256": "97ddaad2b5074dab6f31a5a7a8ded12c3e51123aa5d819f19c006ba123fde089", "input_tokens": 209, "server_ms": 34.85}, {"id": "8a4c3de28be95c105ca1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.481017996091396, "probs": [0.09658208517581993, 0.868022296339126, 0.03539561848505407], "top": "insufficient", "option_logits": null, "prompt_sha256": "5783bf781c97c761780f8d8454e12c8895251116d4d5ebe264401d9d9dcae0a1", "input_tokens": 208, "server_ms": 35.16}, {"id": "ea7ec66a5e8eac886548", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 52.6723510120064, "probs": [0.0651099549445377, 0.02705164492548852, 0.9078384001299739], "top": "prohibited", "option_logits": null, "prompt_sha256": "a92081d320caa09114cab155b336249985672a93e86328503ed4397aa61c3293", "input_tokens": 211, "server_ms": 34.8}, {"id": "25172cd6ddc49ab0ca70", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.352896007709205, "probs": [0.049132112383896635, 0.729412594713007, 0.22145529290309635], "top": "insufficient", "option_logits": null, "prompt_sha256": "2b0f13909e189564a5e5099e9094600d4804ae6ef7ef59e7435f85302977a0c0", "input_tokens": 202, "server_ms": 35.37}, {"id": "fb73a85d386cf59b0c51", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2c3a0cb61f7bae78739e", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.08432597666979, "probs": [0.1638594512396548, 0.04974899459992729, 0.7863915541604178], "top": "prohibited", "option_logits": null, "prompt_sha256": "a4ffa08e79c520845fb80c116412fe4c48304f5f003b514bbcbea0a7c61cfe7b", "input_tokens": 204, "server_ms": 35.1}, {"id": "a02fd582d2881cc462b2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.82585497479886, "probs": [0.06162034871506134, 0.8591822512326693, 0.07919740005226927], "top": "insufficient", "option_logits": null, "prompt_sha256": "f51b6f4e075e67f76ff574d496ba2f9c19cc2ca38a2f56d989791df67251d344", "input_tokens": 214, "server_ms": 35.25}, {"id": "19f37d5a97034bd7d04f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.5947679891251, "probs": [0.7621344563262837, 0.23138990123980285, 0.006475642433913465], "top": "insufficient", "option_logits": null, "prompt_sha256": "baf1674ea5ff54f508700c815711c06e9a50808663840042c147d883c72021ed", "input_tokens": 214, "server_ms": 34.92}, {"id": "fda0ca94652c6d739ecf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.445886995177716, "probs": [0.14398191537109406, 0.021923360098142518, 0.8340947245307634], "top": "insufficient", "option_logits": null, "prompt_sha256": "8854be3865426f818cb7189838d63dc61b9b64da67d13087c8e6ccc2a708a8c0", "input_tokens": 213, "server_ms": 35.42}, {"id": "d0e36c4cb10a29b3191c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "42ed4cbfde7f2f64274b", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 52.992759039625525, "probs": [0.2129289153471605, 0.545676239155193, 0.24139484549764645], "top": "permitted", "option_logits": null, "prompt_sha256": "913468ed99926586c9292a6dabcee9b083a7b1b1c719b6bad31fda7f7f045a2e", "input_tokens": 212, "server_ms": 35.06}, {"id": "a0a39ace7e5337d038a7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.793334984220564, "probs": [0.15943768499721103, 0.025848589203191098, 0.814713725799598], "top": "insufficient", "option_logits": null, "prompt_sha256": "acfb724b1ffcab3d0820c0a1f4cdb189396280c1faf4e622ce1f306cad1bf45e", "input_tokens": 216, "server_ms": 35.42}, {"id": "eeae283841ce75dd25bf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.86720802541822, "probs": [0.9471510047644319, 0.04112297491157286, 0.011726020323995183], "top": "insufficient", "option_logits": null, "prompt_sha256": "7a9df7814cddba377022c022111d998db3d8a2ff48c109eb7f1aac0f5c151319", "input_tokens": 216, "server_ms": 35.05}, {"id": "00bb4cc043f7d2b288d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.92198701156303, "probs": [0.08689224299716879, 0.8314994635810213, 0.08160829342180988], "top": "prohibited", "option_logits": null, "prompt_sha256": "ecd27e22648cba485966b925a1b12fd5a888331b66634b0f8919798224cf8072", "input_tokens": 208, "server_ms": 35.17}, {"id": "1a81ead69e93e5d0c86a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6dd193172b9240a04bf9", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 52.98358196159825, "probs": [0.2729138208779564, 0.3294318709588002, 0.3976543081632435], "top": "insufficient", "option_logits": null, "prompt_sha256": "86759c52a7cc961194391587ca2df8d3998c59ec236637e4acc083f151ad5242", "input_tokens": 215, "server_ms": 35.01}, {"id": "4444dad25e438f128673", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.692355984821916, "probs": [0.5472741025611921, 0.331307576591957, 0.12141832084685092], "top": "prohibited", "option_logits": null, "prompt_sha256": "6ee81091d38ca686274af8a9057ae962583e07c84d4a7c8913246f20a9adc153", "input_tokens": 209, "server_ms": 35.3}, {"id": "3226202ba1ada2570e9f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.43717697542161, "probs": [0.10569064817366915, 0.056437743537509265, 0.8378716082888217], "top": "insufficient", "option_logits": null, "prompt_sha256": "d890e328d92ed0294fd0ca0f45891d715f1b53fae8d48848ee13c936721cc9f2", "input_tokens": 212, "server_ms": 35.07}, {"id": "3f783eec5c90812479bc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 56.51512701297179, "probs": [0.745864682612535, 0.0779433517872937, 0.17619196560017134], "top": "permitted", "option_logits": null, "prompt_sha256": "866975030f9845962316982b48126dfbaa42cd89b8d3719305c5c1cd2a7ce35c", "input_tokens": 209, "server_ms": 37.38}, {"id": "ffee861474a9665a093a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3200eb252d6737e03027", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 54.35496399877593, "probs": [0.30671712087126574, 0.5394587819774863, 0.153824097151248], "top": "permitted", "option_logits": null, "prompt_sha256": "33ed49b156b02a8251a71c89906619f22d3cff122755a441bc701809f452ec03", "input_tokens": 210, "server_ms": 35.65}, {"id": "8c992e17e4fb54b044f5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 54.29646000266075, "probs": [0.9411304198850982, 0.03837677130689544, 0.020492808808006482], "top": "B", "option_logits": null, "prompt_sha256": "26a5967a0d4f86a5e648fdf739a14ae8463a48c7dc0319e4b9026afcfc70ce49", "input_tokens": 200, "server_ms": 35.76}, {"id": "695dd61195ca025c4f4c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 54.33648999314755, "probs": [0.2191677303480843, 0.5616645393038313, 0.2191677303480843], "top": "insufficient", "option_logits": null, "prompt_sha256": "5cb564ef00f9a81eb96ac41fe01780b4b9ffdcb60d7bc8217b9d89860c279ba3", "input_tokens": 201, "server_ms": 35.85}, {"id": "e6155431ab7b0f8170d4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.886410023551434, "probs": [0.8246471790493917, 0.16138164146161363, 0.013971179488994609], "top": "insufficient", "option_logits": null, "prompt_sha256": "e747a01e40c9029fd1004867b0db224405c02f421c64f74cd748920300cb2000", "input_tokens": 192, "server_ms": 35.8}, {"id": "b49ecda75086d8c68d2a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c0cbeb97315307af8111", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.66471299203113, "probs": [0.9136677967042812, 0.07428832300095413, 0.012043880294764588], "top": "B", "option_logits": null, "prompt_sha256": "3d5649d07c298093a047c4d77967454e627c1fd4792f41873c4640c036a163cd", "input_tokens": 192, "server_ms": 35.41}, {"id": "587ced261c6092cd1325", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.93215698515996, "probs": [0.807231387784515, 0.17909249095841714, 0.013676121257067941], "top": "insufficient", "option_logits": null, "prompt_sha256": "88ac81fbf23c9684447ca373863f9cf4c54ca0bd35eb3e960680db4187456d96", "input_tokens": 208, "server_ms": 35.61}, {"id": "533d4423d311de82b27d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.97440999513492, "probs": [0.9655965284054147, 0.028772714439517434, 0.005630757155067839], "top": "insufficient", "option_logits": null, "prompt_sha256": "7263692449eeef173103012af0a80bd27db0a89dc35bf50d4fed73b80a29f045", "input_tokens": 211, "server_ms": 35.26}, {"id": "2bf540d515a51755b25c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.994689020328224, "probs": [0.046119873005046885, 0.9369780144263187, 0.016902112568634386], "top": "B", "option_logits": null, "prompt_sha256": "4c2c0840c1ad7e181d30f10eb62a4618c3195e936034bc7274fe3c8fc945d786", "input_tokens": 200, "server_ms": 35.5}, {"id": "fdb9fc214aea2710763a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "428176cd8ad2c7a74eea", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.841047978494316, "probs": [0.6742384798146065, 0.29826788550402233, 0.02749363468137107], "top": "B", "option_logits": null, "prompt_sha256": "2d0f22653ae25b1b9ad253802809222e0554c7bd42292a5a1637a04128e5c52f", "input_tokens": 190, "server_ms": 35.23}, {"id": "f6d68126b43f8c9ed183", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.14322798606008, "probs": [0.8763745016908776, 0.09751141040687612, 0.026114087902246324], "top": "insufficient", "option_logits": null, "prompt_sha256": "676f3a162c1d11c22fd5f07e5b57e6b6e6d3a4aae515a7c2965aeac124b616dd", "input_tokens": 203, "server_ms": 35.47}, {"id": "44bae0c8a1615a096c43", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.71609599003568, "probs": [0.13968031972950484, 0.23073250176366764, 0.6295871785068274], "top": "insufficient", "option_logits": null, "prompt_sha256": "109526bdb944267ab212d1b8ba2f77a3a6c7c147016cf89ae5bc2e0e8e661adc", "input_tokens": 206, "server_ms": 35.03}, {"id": "2175379f4c69e6207626", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.15766499936581, "probs": [0.9673249441344611, 0.025425185481267775, 0.007249870384270957], "top": "insufficient", "option_logits": null, "prompt_sha256": "c4889b691603599fd8dca223a645f5e88b7248a364a9bef64c37a8a9e3ced5eb", "input_tokens": 197, "server_ms": 35.38}, {"id": "def9fe59d262357a673a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "389e65bf1ba709b84b12", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.78317000577226, "probs": [0.49248819366063284, 0.16951261805505752, 0.3379991882843097], "top": "insufficient", "option_logits": null, "prompt_sha256": "5deb35c130c31229df54964339f9faccffa08f3ed2eb38730d86ac9ccd510aa3", "input_tokens": 192, "server_ms": 35.1}, {"id": "712f6ffb763eef7c9a12", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.02548296842724, "probs": [0.1558415388693792, 0.330859461294187, 0.5132989998364338], "top": "insufficient", "option_logits": null, "prompt_sha256": "7c27c7c3361bd04c584a975218ab6b610dcab0f9bf6078cf3870b470410c383c", "input_tokens": 203, "server_ms": 35.48}, {"id": "e1d610bd14e3d16c09b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.466763987671584, "probs": [0.30165258046080173, 0.13344430167683133, 0.5649031178623669], "top": "B", "option_logits": null, "prompt_sha256": "27dba9d4c38e8b61907893c325f5851e4726ba89fa7ac9e5563a14abe89239b9", "input_tokens": 203, "server_ms": 35.16}, {"id": "22ec57822db50ec121cf", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.63884899998084, "probs": [0.9480355945752582, 0.038658346366163006, 0.013306059058578745], "top": "insufficient", "option_logits": null, "prompt_sha256": "5f3dd4d0a864d976d5c35a7b088b3aed4b55500043acc8701dd4d35729125db8", "input_tokens": 193, "server_ms": 35.33}, {"id": "3e75f2b84e8e08e3d3f2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "45f8ead2522a14384cdc", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.29377198358998, "probs": [0.22389674812152066, 0.7374540645868362, 0.03864918729164314], "top": "A", "option_logits": null, "prompt_sha256": "164c8915823587dee72ef8800598282728e580a02992fe7a8d8fa557ead4e3b9", "input_tokens": 192, "server_ms": 35.22}, {"id": "ebd74aa05324a1556288", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.72011300642043, "probs": [0.9379420530362722, 0.04616732447007806, 0.01589062249364978], "top": "insufficient", "option_logits": null, "prompt_sha256": "d7c03673e95db552913947e72213d55f0938d187ca0d4fd69182cbe141bfda65", "input_tokens": 208, "server_ms": 35.51}, {"id": "d7e84e4245a29817da1f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.952079033479095, "probs": [0.052375134922456754, 0.9385838391059238, 0.009041025971619536], "top": "A", "option_logits": null, "prompt_sha256": "b54ee64662a21acf112153e17512f10436c1938df35ab9ad182be1ff0c47c3e0", "input_tokens": 206, "server_ms": 35.25}, {"id": "f41ae66b3c09630c7a42", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 54.19498903211206, "probs": [0.13579878113602123, 0.8376256951826758, 0.02657552368130303], "top": "A", "option_logits": null, "prompt_sha256": "8aa1b7f158bf57460e6f9f057913e0d01d913f16998cc0a46327e8aa6c93e838", "input_tokens": 200, "server_ms": 35.65}, {"id": "82c2cd97c160ad27951b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3ecec8e2c166e397ad07", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.87283203890547, "probs": [0.25821814633948337, 0.25821814633948337, 0.4835637073210332], "top": "A", "option_logits": null, "prompt_sha256": "b17c06e9ca0e91accd4cfcb4667c5c3a42687d3209e1ecd98910eb1343a18dc2", "input_tokens": 193, "server_ms": 35.27}, {"id": "7ada97b6f99f4788e5c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.118593980092555, "probs": [0.12660014777476133, 0.09251256875552054, 0.7808872834697181], "top": "insufficient", "option_logits": null, "prompt_sha256": "324bb2046ede92322b8175d6af2961ad208eba5f52712692465ef944de2193a4", "input_tokens": 206, "server_ms": 35.09}, {"id": "4d7a003342db5b0a17e3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.86848603282124, "probs": [0.9299397393693557, 0.06263931303365462, 0.007420947596989686], "top": "insufficient", "option_logits": null, "prompt_sha256": "86e7d75bb9b41e2d9de272239185c5cce5c19dc1aac6573a737c31d47ab0563f", "input_tokens": 209, "server_ms": 35.04}, {"id": "19d3d8d0bf773e88be7b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.500056033954024, "probs": [0.37290349783252347, 0.01111172675798736, 0.6159847754094891], "top": "insufficient", "option_logits": null, "prompt_sha256": "a1a07677dc816ad01403f9cff2867d5bfdaff2c29d8a4c2293ce0662a24f33ae", "input_tokens": 194, "server_ms": 35.14}, {"id": "d6de731cca52359b1c87", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a71a17a287f3d30db2aa", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.96016297163442, "probs": [0.475785940079552, 0.1976777825029813, 0.3265362774174668], "top": "insufficient", "option_logits": null, "prompt_sha256": "82b8973ae0b97456180d256ef08822a11d20365d8039148ed863e9bb75c9bda4", "input_tokens": 192, "server_ms": 35.05}, {"id": "b45c63fb92729af27f05", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.54643694590777, "probs": [0.14585095569321507, 0.009226876843769601, 0.8449221674630154], "top": "insufficient", "option_logits": null, "prompt_sha256": "9a672222518ab31da9c3ab2072849ef03b511e8e9d23762f89d33a83feca6ec1", "input_tokens": 216, "server_ms": 35.09}, {"id": "46b7029b9a704138b77a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.359516023192555, "probs": [0.09629593898139954, 0.7633942252849403, 0.14030983573366013], "top": "contradicted", "option_logits": null, "prompt_sha256": "5d2348206184a53d2da714668c8004522fc660c04abc09a4675170278b6d0ce8", "input_tokens": 214, "server_ms": 34.86}, {"id": "caf773b23934fa7362a1", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.0884430045262, "probs": [0.10443660543353643, 0.01402679703707132, 0.8815365975293922], "top": "insufficient", "option_logits": null, "prompt_sha256": "497e689287006b51bd4f178ebc9023050c338fca8abf790eef2112975de7f243", "input_tokens": 209, "server_ms": 35.09}, {"id": "f46f392ef9e9e9df564b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "986d7eaba0c7e9738e79", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.70040802517906, "probs": [0.9369780144263187, 0.046119873005046885, 0.016902112568634386], "top": "supported", "option_logits": null, "prompt_sha256": "984a11157ce42cd37ec6209d96b53bf0d293025d6b69161f1a358f9bcc9d57db", "input_tokens": 207, "server_ms": 35.16}, {"id": "005dcf6d2c0f21d64796", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 53.698587988037616, "probs": [0.6452505747343799, 0.2680863307491953, 0.08666309451642484], "top": "permitted", "option_logits": null, "prompt_sha256": "ea5a71f7bf051b93ca522c5ff3caef6ffdfcb5011f24af51a404b10524b250cb", "input_tokens": 228, "server_ms": 35.55}, {"id": "edb708bd0fe5fd67d6ee", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.581541986204684, "probs": [0.43020843462499875, 0.1903144997976119, 0.3794770655773894], "top": "permitted", "option_logits": null, "prompt_sha256": "b5ac43b527f17cc117db3df096bff98320c5a248422c7b513eedeba99e619731", "input_tokens": 229, "server_ms": 35.32}, {"id": "97a35499bd8717f3e786", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.30395099008456, "probs": [0.056437743537509265, 0.8378716082888217, 0.10569064817366915], "top": "insufficient", "option_logits": null, "prompt_sha256": "be77461c7c49b38137b436f74bcc533cf9fdcc1fbbcabe3acbe8c8c37f2272bd", "input_tokens": 222, "server_ms": 35.22}, {"id": "31eb4533d2c89959ec46", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "ca2e0b043985fa49bcd2", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 53.64675301825628, "probs": [0.1110623891201585, 0.008481107191351438, 0.88045650368849], "top": "permitted", "option_logits": null, "prompt_sha256": "12c28c53461194e05d3490b99aecb157c1965b942f5bfee5ef8163543d2bb1d0", "input_tokens": 221, "server_ms": 35.28}, {"id": "0aa24a6701ac9c56c4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.31027501961216, "probs": [0.7598853370160624, 0.23070704972212444, 0.009407613261813141], "top": "insufficient", "option_logits": null, "prompt_sha256": "0b7bc0317c30767061abcd7c39f9b71942a3c49b230cf334d1e6db7873b4dc61", "input_tokens": 216, "server_ms": 35.15}, {"id": "796d5c0da6eaadae6a3f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.9250269648619, "probs": [0.9718671022270775, 0.019875215614779472, 0.008257682158143088], "top": "B", "option_logits": null, "prompt_sha256": "282f482d7bf4f65f4cc0f8dd6ca05d597f32f80783d49b010173ef79b130ffc5", "input_tokens": 210, "server_ms": 35.09}, {"id": "599b3092b6fd18312d23", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.527687035966665, "probs": [0.08284327606718382, 0.0730741626697933, 0.8440825612630228], "top": "A", "option_logits": null, "prompt_sha256": "6bf8f97ede01d029938070b85d9e3aa545cbbbb84007fb406100f0a0a8ffa474", "input_tokens": 209, "server_ms": 35.14}, {"id": "960a4bcd27363e6db4b9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "85051ba3c30277824ebe", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.50796098355204, "probs": [0.1753450388599366, 0.7903403209860906, 0.03431464015397285], "top": "A", "option_logits": null, "prompt_sha256": "33dc6c8523ece68e76db4eaffbb8a8470964368747977448381041503e3ad36c", "input_tokens": 211, "server_ms": 34.92}, {"id": "c6a411e7f2bbdf377173", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 54.51803799951449, "probs": [0.03629282833642113, 0.01605512937357001, 0.9476520422900088], "top": "supported", "option_logits": null, "prompt_sha256": "6e4467088790119e033569f3066af7eedf2f72c934657e3a20082f685de045f2", "input_tokens": 226, "server_ms": 35.18}, {"id": "e332aa927d0c2ed044aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.5930180340074, "probs": [0.044616056760944024, 0.6623675120954924, 0.2930164311435636], "top": "insufficient", "option_logits": null, "prompt_sha256": "1a718aab5e212abb4604d4e57ad3d7fb8cba756be66eba7d483eec463cb18c6b", "input_tokens": 227, "server_ms": 34.83}, {"id": "a0128e551d095f3449f8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.02555695036426, "probs": [0.04013769079687701, 0.03540454160362492, 0.924457767599498], "top": "contradicted", "option_logits": null, "prompt_sha256": "bafd2feb34ff693920168afa57cc5bc050eea256c82f2e9e071467805a7857f4", "input_tokens": 202, "server_ms": 35.41}, {"id": "46d6a0731bd288c370f0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e7798dab5f4f121cab77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.47908003022894, "probs": [0.21057073282474073, 0.0031468025509083547, 0.7862824646243509], "top": "supported", "option_logits": null, "prompt_sha256": "c4c6394f10ee2df26d3dac75e02000ae56f28719ec24b10add7af6e77f5a4889", "input_tokens": 202, "server_ms": 35.01}, {"id": "21ca7a8b970e36b8e20f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 53.7410459946841, "probs": [0.10502328955939726, 0.2865713245281423, 0.6084053859124604], "top": "permitted", "option_logits": null, "prompt_sha256": "37bd9a494e79ba1e91bbf2a68704609c66421b3ec11ab8a5bdd36908b17135b1", "input_tokens": 233, "server_ms": 35.21}, {"id": "4aa83af95bb69e90bc2d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.4815689898096, "probs": [0.2680863307491953, 0.08666309451642484, 0.6452505747343799], "top": "insufficient", "option_logits": null, "prompt_sha256": "a115c9f1aaee5b00fa1fd3be52a599afa7112fdcf9eec8871ae98b3d6ef2183f", "input_tokens": 231, "server_ms": 35.17}, {"id": "cdfa9eed51fafcf09b72", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 54.23023400362581, "probs": [0.07184163899867897, 0.8298456430479688, 0.09831271795335216], "top": "prohibited", "option_logits": null, "prompt_sha256": "100e5f7a276a412d51e2ddf940777a1ff109ea7803a8291d638760c392064d80", "input_tokens": 216, "server_ms": 35.57}, {"id": "d51b05bfb9d6d7f09331", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "928ebc7817a8f1cac3e4", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "insufficient", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 56.71645904658362, "probs": [0.7469915530638082, 0.14618461002062086, 0.106823836915571], "top": "prohibited", "option_logits": null, "prompt_sha256": "6486b8dc13e88f4d5e489f3b3b76c6bb5aeed63f19005132e10380f0181726aa", "input_tokens": 223, "server_ms": 36.51}, {"id": "fdd902cc6e65fad7cb4f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.48287499509752, "probs": [0.8638851317960057, 0.051328051461926705, 0.08478681674206763], "top": "B", "option_logits": null, "prompt_sha256": "941cd3d853797aa1a92c8261e9339af72809062c590b26ef2db8bdbbc5d68191", "input_tokens": 222, "server_ms": 35.54}, {"id": "42384af9ae56beb37ff5", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.54317801538855, "probs": [0.046253998446132885, 0.9397029314342114, 0.014043070119655722], "top": "A", "option_logits": null, "prompt_sha256": "f9c6ae0da8d46bec4df2b4102f3aebb45748add11af2d63a981d879fea6a717c", "input_tokens": 222, "server_ms": 35.15}, {"id": "cdf7f428857e1b841cb3", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.097658965270966, "probs": [0.04067015270682889, 0.7760155022030047, 0.1833143450901663], "top": "insufficient", "option_logits": null, "prompt_sha256": "4df509501a3fa1e95f377f6b2cd7ff2737ebe52dd6e95472a1a35d0fc43e3fdf", "input_tokens": 215, "server_ms": 35.14}, {"id": "774f7f9245f83c796d44", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "53b0770ec36835b81d3a", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.77067800145596, "probs": [0.05543534738188995, 0.9330136575005004, 0.01155099511760962], "top": "A", "option_logits": null, "prompt_sha256": "d68329bad204407dddf0266c2d55c897537804b87978786e2c29649197957a18", "input_tokens": 201, "server_ms": 35.13}, {"id": "92ec2ffa3dd46ada1335", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.46789601026103, "probs": [0.06878691906304032, 0.03042978117418096, 0.9007832997627787], "top": "supported", "option_logits": null, "prompt_sha256": "3fdb716c314c67c93413daa3cd5a38b13bec265f706d90a97c43e5a108104f13", "input_tokens": 223, "server_ms": 35.21}, {"id": "6149a17bc154f9c5b4a0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 54.270256019663066, "probs": [0.06893683236028562, 0.08321304657583695, 0.8478501210638775], "top": "contradicted", "option_logits": null, "prompt_sha256": "d68195061969b189d71df192da0191d4d56877cf1d8ce0d4479bc8898c998c94", "input_tokens": 219, "server_ms": 35.23}, {"id": "be866e1dbec5d42d9423", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.615732002072036, "probs": [0.016115540131028547, 0.8390475816237557, 0.14483687824521582], "top": "insufficient", "option_logits": null, "prompt_sha256": "6edd6bca63afcb6fb6d263e3e248e24e556fde2b01d965155729342ef37e411e", "input_tokens": 220, "server_ms": 35.22}, {"id": "97d2ed2591fc7638806c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "056cc5b726d6e8300a77", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 52.93986800825223, "probs": [0.2862501142494634, 0.02478133063598072, 0.6889685551145559], "top": "contradicted", "option_logits": null, "prompt_sha256": "d113705714c5c456d527c8466aeda96099d32a8069c13bf0f40b1ff3ed43ad11", "input_tokens": 218, "server_ms": 35.08}, {"id": "4533dd91267b4d815690", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["prohibited"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.79004399199039, "probs": [0.1609098519153101, 0.7722359149508133, 0.06685423313387646], "top": "insufficient", "option_logits": null, "prompt_sha256": "19fd16742b6e54b9e2452e09592de10d5fc44c96cab1630e5f7651b1af77d898", "input_tokens": 224, "server_ms": 35.53}, {"id": "d880e65fef5e2993fdd0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 54.31446497095749, "probs": [0.3043910694191852, 0.39121786116162954, 0.3043910694191852], "top": "insufficient", "option_logits": null, "prompt_sha256": "ea512df6fc353426e37889a53cae2118b9e832157a4af067cb4a85489a5c83ff", "input_tokens": 221, "server_ms": 35.63}, {"id": "90494c76078ffc3c717a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 52.43188398890197, "probs": [0.08097077759111387, 0.04060828666999563, 0.8784209357388906], "top": "prohibited", "option_logits": null, "prompt_sha256": "71ad1455eaea177dccc5d9fc4078b19378e2ce1c1d27289d9120a1c445349487", "input_tokens": 221, "server_ms": 35.06}, {"id": "9ac5cc82c4126c91174a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "e3ce8ec496dbc9974f94", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 53.67804499110207, "probs": [0.8979988326296015, 0.06048782007179681, 0.041513347298601795], "top": "permitted", "option_logits": null, "prompt_sha256": "31939a3c71e268cd4751d57ccd6a07ff44f1ca1f81c7c2f7bf165ea50668fe0a", "input_tokens": 220, "server_ms": 35.37}, {"id": "386e10b1c1ce906473a4", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.36145800538361, "probs": [0.9821934763214742, 0.008345422472445314, 0.009461101206080565], "top": "B", "option_logits": null, "prompt_sha256": "13f4ee2ce11c253ccf784222a18f900b57830384e577f4adf0030d9893ed01a9", "input_tokens": 227, "server_ms": 35.11}, {"id": "4da6aedc1139224eeff2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.01515298197046, "probs": [0.9584447268648851, 0.039082802533792566, 0.0024724706013222815], "top": "insufficient", "option_logits": null, "prompt_sha256": "c8a6bf0c918df98324f40d3c989bf2d3593c8114ebaba9510a0de7a63d90c43f", "input_tokens": 230, "server_ms": 35.43}, {"id": "a22dab8ba073ff6cdb8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.93625604826957, "probs": [0.8152011530118691, 0.1808606650321593, 0.003938181955971634], "top": "insufficient", "option_logits": null, "prompt_sha256": "d512a04940165a8dc92bcf5e8a905fa84148aca8078cba93d34897ca73ebe5fc", "input_tokens": 223, "server_ms": 35.29}, {"id": "42c54fa2ad45743e24d8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a5b234c2e4fb304f1071", "gold": ["insufficient"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "A", "ok": true, "e2e_ms": 53.12767397845164, "probs": [0.9509567466006987, 0.028336479497072638, 0.020706773902228682], "top": "A", "option_logits": null, "prompt_sha256": "726c7f274cbd758a0855ced1b8e06e30b0888e289eeb459d087a868fb9610ef5", "input_tokens": 210, "server_ms": 35.13}, {"id": "f3d13cf184f916677def", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.59909898834303, "probs": [0.03787087428859695, 0.033405034339472804, 0.9287240913719302], "top": "supported", "option_logits": null, "prompt_sha256": "0032df826a7e49e7be6e1c8d5f96ece8822a858eeb6f5280c55a79fe2e628bbb", "input_tokens": 227, "server_ms": 35.16}, {"id": "1105577da4c8dad4609d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.2488549943082, "probs": [0.04458832792770586, 0.7990410839770169, 0.15637058809527726], "top": "insufficient", "option_logits": null, "prompt_sha256": "c29cf273a3d78467e10369969a81a42df358b37cb972ad44f78d455f8785b7e8", "input_tokens": 224, "server_ms": 35.1}, {"id": "0ebd299739302d8b057b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.09558699466288, "probs": [0.6079293570442296, 0.046423581791889615, 0.34564706116388083], "top": "contradicted", "option_logits": null, "prompt_sha256": "a6b9e9fd277a7252eb95c0dea4201d1d75394e8dd03ee09f576f6fc0355881f3", "input_tokens": 220, "server_ms": 35.1}, {"id": "eafc22c8c40df3932a8e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "d142f7fd0efa0eddc788", "gold": ["insufficient"], "forbid": [], "option_ids": ["supported", "contradicted", "insufficient"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 52.376026986166835, "probs": [0.07421350543321283, 0.7101726109045401, 0.2156138836622472], "top": "contradicted", "option_logits": null, "prompt_sha256": "25fc9b80402930a472ddf5fbb9527f22c905747ffefc59e4865da0bcf422e5bb", "input_tokens": 208, "server_ms": 34.88}, {"id": "0aa1447680e4f25862bb", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["prohibited"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.54632595460862, "probs": [0.3596689227288795, 0.5240624893078483, 0.11626858796327226], "top": "prohibited", "option_logits": null, "prompt_sha256": "1a9834724cc432424395c6e3e4989cae7698e00f85fedac3d9a8cd1a698cd265", "input_tokens": 227, "server_ms": 35.08}, {"id": "980e76145c925747220c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 52.988498006016016, "probs": [0.17244943152985545, 0.1836151222987057, 0.6439354461714388], "top": "permitted", "option_logits": null, "prompt_sha256": "cc59abf9dfd3346910e8d93dc78fe4cd21d0c20962370b801e609b82712c7ea7", "input_tokens": 228, "server_ms": 34.87}, {"id": "4aee37a6a155747766aa", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.53621400333941, "probs": [0.11048645425133781, 0.06688603847853027, 0.8226275072701319], "top": "insufficient", "option_logits": null, "prompt_sha256": "45e75124dce70ee83cbbd2df43d7c5f3045b0da83559f8b4772e8cb6fa82dbb0", "input_tokens": 225, "server_ms": 35.03}, {"id": "6db17cc22bd656d78558", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "3476d3f4f32290fe57c3", "gold": ["insufficient"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 52.6927980245091, "probs": [0.07600853330506088, 0.8779775900702179, 0.046013876624721124], "top": "permitted", "option_logits": null, "prompt_sha256": "902f5bf4dd52878d854ff660af610beb5cd00553cb6cf9fa5c1e7a5ea010391b", "input_tokens": 218, "server_ms": 35.04}, {"id": "ef3e0f8e657bf59f8a29", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.29592601628974, "probs": [0.07014840077352269, 0.16883849753435962, 0.7610131016921177], "top": "B", "option_logits": null, "prompt_sha256": "d4a4f3c22ce6befbff78058a364e4617bda379057c95b5bacf2410a162ea27f7", "input_tokens": 221, "server_ms": 35.36}, {"id": "f756e1cb04e11766341e", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["insufficient", "A", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.40414303261787, "probs": [0.7195969840802272, 0.2637194956142592, 0.016683520305513583], "top": "insufficient", "option_logits": null, "prompt_sha256": "fc0371a2d78013d514b134511d38b37de6e71a39ad2fa9b95b0119945db7a316", "input_tokens": 227, "server_ms": 34.86}, {"id": "d167e51bb1f499099645", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["A"], "forbid": [], "option_ids": ["A", "B", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 57.52971401670948, "probs": [0.6328238128438389, 0.029254643239392513, 0.33792154391676854], "top": "A", "option_logits": null, "prompt_sha256": "5521ebbd2304bdf6049132465a2b8c7b28c862f462f50f98d90efeb483601e64", "input_tokens": 207, "server_ms": 35.36}, {"id": "674ca4e9346216255466", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "c979db7416b206c1eb09", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 55.29005505377427, "probs": [0.22351946247571317, 0.12708523693891682, 0.64939530058537], "top": "A", "option_logits": null, "prompt_sha256": "c11ad01ccd57e2b3b849c07b89887124412bce4a8e5506adaeb954f23c598ac7", "input_tokens": 197, "server_ms": 35.07}, {"id": "1df851a202994a07f123", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["contradicted"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.25377004919574, "probs": [0.038658346366163006, 0.013306059058578745, 0.9480355945752582], "top": "supported", "option_logits": null, "prompt_sha256": "8d5e868c2065409c5dcedc10c1d875c102dfbb48d2433b234f0ad0456f1049bc", "input_tokens": 213, "server_ms": 35.27}, {"id": "b5e6788d95b8395d2888", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.24280296918005, "probs": [0.01953747345276732, 0.9553520377742281, 0.025110488773004678], "top": "contradicted", "option_logits": null, "prompt_sha256": "6ac7738979b2b31bef2f7e2f3d8984b0b2b559ee71fa4d2b728e28884e3183b5", "input_tokens": 212, "server_ms": 34.78}, {"id": "32f36cd0fbbfb3604586", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["supported"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "contradicted", "ok": true, "e2e_ms": 53.813372971490026, "probs": [0.05100292527189731, 0.035003776957469146, 0.9139932977706335], "top": "contradicted", "option_logits": null, "prompt_sha256": "66c4b4c8b3dc52b623b51acf50a373bf625ad5559da7c3bf90c47f56e7c2829b", "input_tokens": 202, "server_ms": 35.37}, {"id": "843185706adff0c60f8b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "f7403e39d6796072491c", "gold": ["insufficient"], "forbid": [], "option_ids": ["contradicted", "supported", "insufficient"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 52.73594200843945, "probs": [0.09842495451008554, 0.01699017293024806, 0.8845848725596664], "top": "insufficient", "option_logits": null, "prompt_sha256": "7db376c4d8c96d2c69fda861429911890ec2d70b0f375e60f65c846f1693e18a", "input_tokens": 203, "server_ms": 35.16}, {"id": "b531dd603bf080dab820", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.92250698059797, "probs": [0.1344819107818076, 0.03601499116251437, 0.829503098055678], "top": "insufficient", "option_logits": null, "prompt_sha256": "03449c2d41ee64f00d22b4b6e619aa69ce6d623d00b4972f503f5ed336bba08e", "input_tokens": 218, "server_ms": 35.49}, {"id": "be2dd67bc733c1213bef", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 52.86735499976203, "probs": [0.9331701885712249, 0.04890646327936172, 0.017923348149413328], "top": "prohibited", "option_logits": null, "prompt_sha256": "2ed1d2ff423bdb9e622336508299a3180255754a84d1847b9b5ba5d1d5623692", "input_tokens": 221, "server_ms": 34.98}, {"id": "b8e2750307df9761e716", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["permitted"], "forbid": [], "option_ids": ["permitted", "prohibited", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.91836697142571, "probs": [0.05755147392994642, 0.0327217252711575, 0.9097268007988961], "top": "insufficient", "option_logits": null, "prompt_sha256": "fb0073f867c77a04f9287adbd88276517a2a85e5cf9dcfb04c81ff5ba768e03f", "input_tokens": 215, "server_ms": 35.03}, {"id": "5d31b4bf2d0fc193f53b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "a060fd4621be415e9f1a", "gold": ["insufficient"], "forbid": [], "option_ids": ["permitted", "insufficient", "prohibited"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 52.53435199847445, "probs": [0.08284327606718382, 0.8440825612630228, 0.0730741626697933], "top": "insufficient", "option_logits": null, "prompt_sha256": "51c4ac799c31ceb752046516252cb8d59c3a3c1654c685c82476530a4d74a689", "input_tokens": 220, "server_ms": 34.99}, {"id": "a9230edfc6ec9e524af0", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["A"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.999076946638525, "probs": [0.040967527214491276, 0.8323022347202245, 0.12673023806528433], "top": "insufficient", "option_logits": null, "prompt_sha256": "2a5a9b08e7ca06d1f5808cdcd3270c79b064cd4fed16b4be278fc7d5f58a1d1f", "input_tokens": 206, "server_ms": 35.23}, {"id": "c100e84585149a04be49", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.56508704042062, "probs": [0.048847636440916636, 0.07578275550521898, 0.8753696080538644], "top": "insufficient", "option_logits": null, "prompt_sha256": "555fd1e019506f048448ae23a6bac4fe7f3e9f2fe60181be02acfdf8225318e5", "input_tokens": 212, "server_ms": 34.9}, {"id": "579beed184ffc1104aa9", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["B"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.71377803385258, "probs": [0.07319171247064139, 0.2264135372721482, 0.7003947502572104], "top": "insufficient", "option_logits": null, "prompt_sha256": "1fc713f5dfc86b0d90f75094b087d920fbfeb9830fb4eec74f78d2f1d67fbf66", "input_tokens": 203, "server_ms": 35.2}, {"id": "c4e9bd463b6fde5be743", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2b0f8eda8903e38a47de", "gold": ["insufficient"], "forbid": [], "option_ids": ["B", "insufficient", "A"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.373233997263014, "probs": [0.9387496456243781, 0.055776151759141294, 0.005474202616480596], "top": "B", "option_logits": null, "prompt_sha256": "77e6633b56fd111f3cf38ff30d1c308c084b3de608efb9682cf02409dc31c9db", "input_tokens": 191, "server_ms": 35.28}, {"id": "b863998d4be2721a5dfd", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["contradicted"], "forbid": [], "option_ids": ["supported", "insufficient", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.50263399304822, "probs": [0.46903312718147117, 0.38856479455009935, 0.14240207826842943], "top": "supported", "option_logits": null, "prompt_sha256": "83104242d8900e7e51e6ca6f99d29588121dd7153c0fda9d487e3eafef91196e", "input_tokens": 224, "server_ms": 35.39}, {"id": "37cebc3792e98049439f", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["insufficient", "supported", "contradicted"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 54.20296499505639, "probs": [0.7720397533404616, 0.11755438599864407, 0.11040586066089443], "top": "insufficient", "option_logits": null, "prompt_sha256": "66f0b52c1eee120f16069ed75fbd7bfc556b5a3dc09c27f809799b32ae74e959", "input_tokens": 221, "server_ms": 35.4}, {"id": "64d63f86e328aff1481c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["supported"], "forbid": [], "option_ids": ["contradicted", "insufficient", "supported"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.23905003024265, "probs": [0.014236196905258059, 0.6140385266223682, 0.3717252764723737], "top": "insufficient", "option_logits": null, "prompt_sha256": "fac76f29223ea6573fb84bb8cb70a27292765e7f4f90ebb19001e8a0b7bf0533", "input_tokens": 229, "server_ms": 35.15}, {"id": "6d64a8459f64eac7f4cc", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "01020a1eaf46ec2fba5d", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "contradicted", "supported"], "gold_desc_id": "supported", "ok": true, "e2e_ms": 53.044970030896366, "probs": [0.06291220787817095, 0.0030966605414949147, 0.9339911315803341], "top": "supported", "option_logits": null, "prompt_sha256": "89df7654ae643d961d5bd0a1cf472717b674fe14a8f5377a6918c1103303e14f", "input_tokens": 226, "server_ms": 35.22}, {"id": "2708193212d8a4a523c7", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["prohibited"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.82668903237209, "probs": [0.5216500126916775, 0.26161676918070537, 0.21673321812761712], "top": "prohibited", "option_logits": null, "prompt_sha256": "73a309dd6a1605e1d928e2a4947516f3ae54fda513d77cea9d4fdf892dbbebb5", "input_tokens": 226, "server_ms": 35.36}, {"id": "5a37d1dfcad20810a76a", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["prohibited", "permitted", "insufficient"], "gold_desc_id": "prohibited", "ok": true, "e2e_ms": 53.53896098677069, "probs": [0.843310768581531, 0.1455727910071266, 0.01111644041134243], "top": "prohibited", "option_logits": null, "prompt_sha256": "97510bceb98f70589de7351d3f810c2e4c175b03bbf6ddec55eca00ef9c199ee", "input_tokens": 215, "server_ms": 35.43}, {"id": "acacd8b46acd25cb954b", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["permitted"], "forbid": [], "option_ids": ["insufficient", "permitted", "prohibited"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 53.20478504290804, "probs": [0.42929779219364844, 0.486689574582582, 0.08401263322376958], "top": "permitted", "option_logits": null, "prompt_sha256": "5a1d1ca84275c476cd245cafcd0e1fd36598a6dae3c643b8ea9d60c5b57d6535", "input_tokens": 225, "server_ms": 35.18}, {"id": "4089c199b8b796ec3ed8", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "6a06a2f278301c1412ee", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "prohibited", "permitted"], "gold_desc_id": "permitted", "ok": true, "e2e_ms": 52.63049900531769, "probs": [0.19560247361171754, 0.23611000151056785, 0.5682875248777147], "top": "permitted", "option_logits": null, "prompt_sha256": "fec6fd7a8c54ee7856b56b8cc85aa3ff4cb617820fd52f7faf204e2efb4a86ff", "input_tokens": 224, "server_ms": 35.11}, {"id": "02f2b95a5c5a4777a45c", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["A"], "forbid": [], "option_ids": ["B", "A", "insufficient"], "gold_desc_id": "B", "ok": true, "e2e_ms": 53.67895495146513, "probs": [0.9323898466010938, 0.02450695094155103, 0.04310320245735524], "top": "B", "option_logits": null, "prompt_sha256": "fcc6cca56205c973fe5ee42de7bbca225406234b5346f63e55af7d8c5f115d9a", "input_tokens": 213, "server_ms": 35.27}, {"id": "ebea4681e3dcc7d92ca2", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.97836201498285, "probs": [0.643833389199536, 0.3438005390976864, 0.012366071702777563], "top": "insufficient", "option_logits": null, "prompt_sha256": "176f3f577d5802518ba6bb8a36039f8c2c7fa4991aac2e236870e0189d0b31a6", "input_tokens": 218, "server_ms": 34.85}, {"id": "b8c5b9b285cbdddc9f8d", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["B"], "forbid": [], "option_ids": ["A", "insufficient", "B"], "gold_desc_id": "insufficient", "ok": true, "e2e_ms": 52.64011799590662, "probs": [0.020042268594912593, 0.21743089555343206, 0.7625268358516554], "top": "B", "option_logits": null, "prompt_sha256": "8494413c37184f76cf07859eef5fe11f7373b1acb0da7cbd005226a5bb00610d", "input_tokens": 209, "server_ms": 35.08}, {"id": "7ed75d854d99b3a81916", "set": "authored144", "cond": "evidence", "decision": "evidence", "tag": "case", "group": "2797f8c23d05c0ee06b3", "gold": ["insufficient"], "forbid": [], "option_ids": ["insufficient", "B", "A"], "gold_desc_id": "A", "ok": true, "e2e_ms": 52.985697984695435, "probs": [0.33058297459294167, 0.050336110784327995, 0.6190809146227304], "top": "A", "option_logits": null, "prompt_sha256": "70f353d85ca31feb6cd6d5c421fe7a2e877a313657d44902068dc2f6ef451bf8", "input_tokens": 207, "server_ms": 34.65}], "seconds": 7.7, "failures": 0}}, "finished_utc": "2026-09-30T16:46:49Z"}