{
 "report": {
  "scenario": "cicada-gate-w1-context",
  "cases": 35,
  "runs": 3,
  "latency_per_turn": {
   "e2e_ms": {
    "median": 105.2,
    "min": 103.2,
    "max": 125.4,
    "n": 105
   },
   "srv_ms": {
    "median": 76.1,
    "min": 75.3,
    "max": 78.1,
    "n": 105
   }
  },
  "by_decision": {
   "gate": {
    "evidence": {
     "n": 31,
     "correct": 30,
     "acc": 0.968
    },
    "blind_null": {
     "n": 31,
     "correct": 16,
     "acc": 0.516
    },
    "controls": {
     "n": 5,
     "correct": 5,
     "acc": 1.0
    },
    "non_controls": {
     "n": 26,
     "correct": 25,
     "acc": 0.962
    },
    "by_label": {
     "ordinary": {
      "evidence": {
       "n": 16,
       "correct": 16,
       "acc": 1.0
      },
      "blind_null": {
       "n": 16,
       "correct": 16,
       "acc": 1.0
      }
     },
     "react": {
      "evidence": {
       "n": 15,
       "correct": 14,
       "acc": 0.933
      },
      "blind_null": {
       "n": 15,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 30,
     "correct": 29,
     "acc": 0.967
    },
    "split": {
     "n": 1,
     "correct": 1,
     "acc": 1.0
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "cheese: said ordinary (p=0.835, agree=1.0) want react"
    ]
   }
  }
 },
 "rows": [
  {
   "case": "lights-off",
   "tag": "control",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.844,
   "agreement": 1.0,
   "probs": {
    "react": 0.156,
    "ordinary": 0.844
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "timer",
   "tag": "control",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.818,
   "agreement": 1.0,
   "probs": {
    "react": 0.182,
    "ordinary": 0.818
   },
   "blind_top": "ordinary",
   "input_tokens": 187,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "weather",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.915,
   "agreement": 1.0,
   "probs": {
    "react": 0.085,
    "ordinary": 0.915
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "weather-followup",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.531,
   "agreement": 0.5,
   "probs": {
    "react": 0.469,
    "ordinary": 0.531
   },
   "blind_top": "ordinary",
   "input_tokens": 198,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "add-milk",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.899,
   "agreement": 1.0,
   "probs": {
    "react": 0.101,
    "ordinary": 0.899
   },
   "blind_top": "ordinary",
   "input_tokens": 187,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "thermostat",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.665,
   "agreement": 1.0,
   "probs": {
    "react": 0.335,
    "ordinary": 0.665
   },
   "blind_top": "ordinary",
   "input_tokens": 188,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "what-time",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.95,
   "agreement": 1.0,
   "probs": {
    "react": 0.05,
    "ordinary": 0.95
   },
   "blind_top": "ordinary",
   "input_tokens": 185,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "tablespoons",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.963,
   "agreement": 1.0,
   "probs": {
    "react": 0.037,
    "ordinary": 0.963
   },
   "blind_top": "ordinary",
   "input_tokens": 187,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "times-table",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.899,
   "agreement": 1.0,
   "probs": {
    "react": 0.101,
    "ordinary": 0.899
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "recipe-step",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.766,
   "agreement": 1.0,
   "probs": {
    "react": 0.234,
    "ordinary": 0.766
   },
   "blind_top": "ordinary",
   "input_tokens": 207,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "soup",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.766,
   "agreement": 1.0,
   "probs": {
    "react": 0.234,
    "ordinary": 0.766
   },
   "blind_top": "ordinary",
   "input_tokens": 213,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "play-music",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.651,
   "agreement": 1.0,
   "probs": {
    "react": 0.349,
    "ordinary": 0.651
   },
   "blind_top": "ordinary",
   "input_tokens": 184,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "dentist",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.92,
   "agreement": 1.0,
   "probs": {
    "react": 0.08,
    "ordinary": 0.92
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "spell",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.808,
   "agreement": 1.0,
   "probs": {
    "react": 0.192,
    "ordinary": 0.808
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "remind",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.91,
   "agreement": 1.0,
   "probs": {
    "react": 0.09,
    "ordinary": 0.91
   },
   "blind_top": "ordinary",
   "input_tokens": 189,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "volume",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.743,
   "agreement": 1.0,
   "probs": {
    "react": 0.257,
    "ordinary": 0.743
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "dog-died",
   "tag": "control",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.99,
   "agreement": 1.0,
   "probs": {
    "react": 0.99,
    "ordinary": 0.01
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "got-job",
   "tag": "control",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.995,
   "agreement": 1.0,
   "probs": {
    "react": 0.995,
    "ordinary": 0.005
   },
   "blind_top": "ordinary",
   "input_tokens": 193,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "intruder",
   "tag": "control",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.985,
   "agreement": 1.0,
   "probs": {
    "react": 0.985,
    "ordinary": 0.015
   },
   "blind_top": "ordinary",
   "input_tokens": 226,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "fish",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.915,
   "agreement": 1.0,
   "probs": {
    "react": 0.915,
    "ordinary": 0.085
   },
   "blind_top": "ordinary",
   "input_tokens": 193,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "hospital",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.969,
   "agreement": 1.0,
   "probs": {
    "react": 0.969,
    "ordinary": 0.031
   },
   "blind_top": "ordinary",
   "input_tokens": 188,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "laid-off",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.99,
   "agreement": 1.0,
   "probs": {
    "react": 0.99,
    "ordinary": 0.01
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "baby-steps",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.974,
   "agreement": 1.0,
   "probs": {
    "react": 0.974,
    "ordinary": 0.026
   },
   "blind_top": "ordinary",
   "input_tokens": 188,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "cut-finger",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.972,
   "agreement": 1.0,
   "probs": {
    "react": 0.972,
    "ordinary": 0.028
   },
   "blind_top": "ordinary",
   "input_tokens": 191,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "raccoon",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.965,
   "agreement": 1.0,
   "probs": {
    "react": 0.965,
    "ordinary": 0.035
   },
   "blind_top": "ordinary",
   "input_tokens": 191,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "snow-june",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.874,
   "agreement": 1.0,
   "probs": {
    "react": 0.874,
    "ordinary": 0.126
   },
   "blind_top": "ordinary",
   "input_tokens": 189,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "toaster",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.777,
   "agreement": 1.0,
   "probs": {
    "react": 0.777,
    "ordinary": 0.223
   },
   "blind_top": "ordinary",
   "input_tokens": 191,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "scarecrow",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.755,
   "agreement": 1.0,
   "probs": {
    "react": 0.755,
    "ordinary": 0.245
   },
   "blind_top": "ordinary",
   "input_tokens": 197,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "cookie",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.562,
   "agreement": 1.0,
   "probs": {
    "react": 0.562,
    "ordinary": 0.438
   },
   "blind_top": "ordinary",
   "input_tokens": 189,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "useless",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.932,
   "agreement": 1.0,
   "probs": {
    "react": 0.932,
    "ordinary": 0.068
   },
   "blind_top": "ordinary",
   "input_tokens": 190,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "cheese",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.835,
   "agreement": 1.0,
   "probs": {
    "react": 0.165,
    "ordinary": 0.835
   },
   "blind_top": "ordinary",
   "input_tokens": 187,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "goodnight",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "react",
   "p_top": 0.577,
   "agreement": 1.0,
   "probs": {
    "react": 0.577,
    "ordinary": 0.423
   },
   "blind_top": "ordinary",
   "input_tokens": 186,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "thanks",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "react",
   "p_top": 0.893,
   "agreement": 1.0,
   "probs": {
    "react": 0.893,
    "ordinary": 0.107
   },
   "blind_top": "ordinary",
   "input_tokens": 190,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "sit-with-me",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "react",
   "p_top": 0.562,
   "agreement": 1.0,
   "probs": {
    "react": 0.562,
    "ordinary": 0.438
   },
   "blind_top": "ordinary",
   "input_tokens": 212,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "stop-timer",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "ordinary",
   "p_top": 0.622,
   "agreement": 1.0,
   "probs": {
    "react": 0.378,
    "ordinary": 0.622
   },
   "blind_top": "ordinary",
   "input_tokens": 197,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  }
 ]
}