{
 "report": {
  "scenario": "cicada-gate-w2-context",
  "cases": 35,
  "runs": 3,
  "latency_per_turn": {
   "e2e_ms": {
    "median": 104.6,
    "min": 102.9,
    "max": 130.9,
    "n": 105
   },
   "srv_ms": {
    "median": 75.5,
    "min": 74.7,
    "max": 79.5,
    "n": 105
   }
  },
  "by_decision": {
   "gate": {
    "evidence": {
     "n": 31,
     "correct": 19,
     "acc": 0.613
    },
    "blind_null": {
     "n": 31,
     "correct": 16,
     "acc": 0.516
    },
    "controls": {
     "n": 5,
     "correct": 4,
     "acc": 0.8
    },
    "non_controls": {
     "n": 26,
     "correct": 15,
     "acc": 0.577
    },
    "by_label": {
     "ordinary": {
      "evidence": {
       "n": 16,
       "correct": 16,
       "acc": 1.0
      },
      "blind_null": {
       "n": 16,
       "correct": 16,
       "acc": 1.0
      }
     },
     "react": {
      "evidence": {
       "n": 15,
       "correct": 3,
       "acc": 0.2
      },
      "blind_null": {
       "n": 15,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 30,
     "correct": 19,
     "acc": 0.633
    },
    "split": {
     "n": 1,
     "correct": 0,
     "acc": 0.0
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "dog-died: said ordinary (p=0.928, agree=1.0) want react",
     "fish: said ordinary (p=0.651, agree=1.0) want react",
     "hospital: said ordinary (p=0.651, agree=1.0) want react",
     "laid-off: said ordinary (p=0.852, agree=1.0) want react",
     "cut-finger: said ordinary (p=0.562, agree=0.5) want react",
     "raccoon: said ordinary (p=0.808, agree=1.0) want react",
     "snow-june: said ordinary (p=0.974, agree=1.0) want react",
     "toaster: said ordinary (p=0.994, agree=1.0) want react",
     "scarecrow: said ordinary (p=0.915, agree=1.0) want react",
     "cookie: said ordinary (p=0.947, agree=1.0) want react",
     "useless: said ordinary (p=0.987, agree=1.0) want react",
     "cheese: said ordinary (p=0.997, agree=1.0) want react"
    ]
   }
  }
 },
 "rows": [
  {
   "case": "lights-off",
   "tag": "control",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.887,
   "agreement": 1.0,
   "probs": {
    "react": 0.113,
    "ordinary": 0.887
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "timer",
   "tag": "control",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.991,
   "agreement": 1.0,
   "probs": {
    "react": 0.009,
    "ordinary": 0.991
   },
   "blind_top": "ordinary",
   "input_tokens": 113,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "weather",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.994,
   "agreement": 1.0,
   "probs": {
    "react": 0.006,
    "ordinary": 0.994
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "weather-followup",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.881,
   "agreement": 1.0,
   "probs": {
    "react": 0.119,
    "ordinary": 0.881
   },
   "blind_top": "ordinary",
   "input_tokens": 124,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "add-milk",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.971,
   "agreement": 1.0,
   "probs": {
    "react": 0.029,
    "ordinary": 0.971
   },
   "blind_top": "ordinary",
   "input_tokens": 113,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "thermostat",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.844,
   "agreement": 1.0,
   "probs": {
    "react": 0.156,
    "ordinary": 0.844
   },
   "blind_top": "ordinary",
   "input_tokens": 114,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "what-time",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.992,
   "agreement": 1.0,
   "probs": {
    "react": 0.008,
    "ordinary": 0.992
   },
   "blind_top": "ordinary",
   "input_tokens": 111,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "tablespoons",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.998,
   "agreement": 1.0,
   "probs": {
    "react": 0.002,
    "ordinary": 0.998
   },
   "blind_top": "ordinary",
   "input_tokens": 113,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "times-table",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "react": 0.001,
    "ordinary": 0.999
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "recipe-step",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.981,
   "agreement": 1.0,
   "probs": {
    "react": 0.019,
    "ordinary": 0.981
   },
   "blind_top": "ordinary",
   "input_tokens": 133,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "soup",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.743,
   "agreement": 1.0,
   "probs": {
    "react": 0.257,
    "ordinary": 0.743
   },
   "blind_top": "ordinary",
   "input_tokens": 139,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "play-music",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.915,
   "agreement": 1.0,
   "probs": {
    "react": 0.085,
    "ordinary": 0.915
   },
   "blind_top": "ordinary",
   "input_tokens": 110,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "dentist",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.983,
   "agreement": 1.0,
   "probs": {
    "react": 0.017,
    "ordinary": 0.983
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "spell",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.994,
   "agreement": 1.0,
   "probs": {
    "react": 0.006,
    "ordinary": 0.994
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "remind",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.98,
   "agreement": 1.0,
   "probs": {
    "react": 0.02,
    "ordinary": 0.98
   },
   "blind_top": "ordinary",
   "input_tokens": 115,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "volume",
   "tag": "case",
   "decision": "gate",
   "label": "ordinary",
   "top": "ordinary",
   "p_top": 0.852,
   "agreement": 1.0,
   "probs": {
    "react": 0.148,
    "ordinary": 0.852
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "dog-died",
   "tag": "control",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.928,
   "agreement": 1.0,
   "probs": {
    "react": 0.072,
    "ordinary": 0.928
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "got-job",
   "tag": "control",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.905,
   "agreement": 1.0,
   "probs": {
    "react": 0.905,
    "ordinary": 0.095
   },
   "blind_top": "ordinary",
   "input_tokens": 119,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "intruder",
   "tag": "control",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.955,
   "agreement": 1.0,
   "probs": {
    "react": 0.955,
    "ordinary": 0.045
   },
   "blind_top": "ordinary",
   "input_tokens": 152,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "fish",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.651,
   "agreement": 1.0,
   "probs": {
    "react": 0.349,
    "ordinary": 0.651
   },
   "blind_top": "ordinary",
   "input_tokens": 119,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "hospital",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.651,
   "agreement": 1.0,
   "probs": {
    "react": 0.349,
    "ordinary": 0.651
   },
   "blind_top": "ordinary",
   "input_tokens": 114,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "laid-off",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.852,
   "agreement": 1.0,
   "probs": {
    "react": 0.148,
    "ordinary": 0.852
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "baby-steps",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "react",
   "p_top": 0.731,
   "agreement": 1.0,
   "probs": {
    "react": 0.731,
    "ordinary": 0.269
   },
   "blind_top": "ordinary",
   "input_tokens": 114,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "cut-finger",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.562,
   "agreement": 0.5,
   "probs": {
    "react": 0.438,
    "ordinary": 0.562
   },
   "blind_top": "ordinary",
   "input_tokens": 117,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "raccoon",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.808,
   "agreement": 1.0,
   "probs": {
    "react": 0.192,
    "ordinary": 0.808
   },
   "blind_top": "ordinary",
   "input_tokens": 117,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "snow-june",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.974,
   "agreement": 1.0,
   "probs": {
    "react": 0.026,
    "ordinary": 0.974
   },
   "blind_top": "ordinary",
   "input_tokens": 115,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "toaster",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.994,
   "agreement": 1.0,
   "probs": {
    "react": 0.006,
    "ordinary": 0.994
   },
   "blind_top": "ordinary",
   "input_tokens": 117,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "scarecrow",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.915,
   "agreement": 1.0,
   "probs": {
    "react": 0.085,
    "ordinary": 0.915
   },
   "blind_top": "ordinary",
   "input_tokens": 123,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "cookie",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.947,
   "agreement": 1.0,
   "probs": {
    "react": 0.053,
    "ordinary": 0.947
   },
   "blind_top": "ordinary",
   "input_tokens": 115,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "useless",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.987,
   "agreement": 1.0,
   "probs": {
    "react": 0.013,
    "ordinary": 0.987
   },
   "blind_top": "ordinary",
   "input_tokens": 116,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "cheese",
   "tag": "case",
   "decision": "gate",
   "label": "react",
   "top": "ordinary",
   "p_top": 0.997,
   "agreement": 1.0,
   "probs": {
    "react": 0.003,
    "ordinary": 0.997
   },
   "blind_top": "ordinary",
   "input_tokens": 113,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "goodnight",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "ordinary",
   "p_top": 0.936,
   "agreement": 1.0,
   "probs": {
    "react": 0.064,
    "ordinary": 0.936
   },
   "blind_top": "ordinary",
   "input_tokens": 112,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "thanks",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "ordinary",
   "p_top": 0.693,
   "agreement": 1.0,
   "probs": {
    "react": 0.307,
    "ordinary": 0.693
   },
   "blind_top": "ordinary",
   "input_tokens": 116,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "sit-with-me",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "react",
   "p_top": 0.706,
   "agreement": 1.0,
   "probs": {
    "react": 0.706,
    "ordinary": 0.294
   },
   "blind_top": "ordinary",
   "input_tokens": 138,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "stop-timer",
   "tag": "open",
   "decision": "gate",
   "label": null,
   "top": "ordinary",
   "p_top": 0.844,
   "agreement": 1.0,
   "probs": {
    "react": 0.156,
    "ordinary": 0.844
   },
   "blind_top": "ordinary",
   "input_tokens": 123,
   "ok": null,
   "blind_ok": null,
   "egregious": false,
   "blind_egregious": false
  }
 ]
}