{
 "report": {
  "scenario": "wyrd-scene-toll",
  "cases": 7,
  "runs": 3,
  "latency_per_turn": {
   "e2e_ms": {
    "median": 137.7,
    "min": 136.4,
    "max": 766.8,
    "n": 21
   },
   "srv_ms": {
    "median": 106.5,
    "min": 105.8,
    "max": 706.7,
    "n": 21
   }
  },
  "by_decision": {
   "place": {
    "evidence": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "blind_null": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "controls": {
     "n": 2,
     "correct": 1,
     "acc": 0.5
    },
    "non_controls": {
     "n": 5,
     "correct": 4,
     "acc": 0.8
    },
    "by_label": {
     "holds": {
      "evidence": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     },
     "moved": {
      "evidence": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "t-granted: said holds (p=0.991, agree=1.0) want moved",
     "t-denied: said holds (p=0.978, agree=1.0) want moved"
    ]
   },
   "exit": {
    "evidence": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "blind_null": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "controls": {
     "n": 2,
     "correct": 1,
     "acc": 0.5
    },
    "non_controls": {
     "n": 5,
     "correct": 4,
     "acc": 0.8
    },
    "by_label": {
     "departure": {
      "evidence": {
       "n": 2,
       "correct": 2,
       "acc": 1.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "stay": {
      "evidence": {
       "n": 5,
       "correct": 3,
       "acc": 0.6
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 4,
     "correct": 3,
     "acc": 0.75
    },
    "split": {
     "n": 3,
     "correct": 2,
     "acc": 0.667
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "t-reads-letter: said departure (p=0.593, agree=0.5) want stay",
     "t-questions: said departure (p=0.665, agree=1.0) want stay"
    ]
   },
   "place2": {
    "evidence": {
     "n": 7,
     "correct": 7,
     "acc": 1.0
    },
    "blind_null": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "controls": {
     "n": 2,
     "correct": 2,
     "acc": 1.0
    },
    "non_controls": {
     "n": 5,
     "correct": 5,
     "acc": 1.0
    },
    "by_label": {
     "elsewhere": {
      "evidence": {
       "n": 2,
       "correct": 2,
       "acc": 1.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "same": {
      "evidence": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 7,
     "correct": 7,
     "acc": 1.0
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": []
   },
   "exit2": {
    "evidence": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "blind_null": {
     "n": 7,
     "correct": 5,
     "acc": 0.714
    },
    "controls": {
     "n": 2,
     "correct": 1,
     "acc": 0.5
    },
    "non_controls": {
     "n": 5,
     "correct": 4,
     "acc": 0.8
    },
    "by_label": {
     "departure": {
      "evidence": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "stay": {
      "evidence": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 6,
     "correct": 5,
     "acc": 0.833
    },
    "split": {
     "n": 1,
     "correct": 0,
     "acc": 0.0
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "t-granted: said stay (p=0.978, agree=1.0) want departure",
     "t-denied: said stay (p=0.637, agree=0.5) want departure"
    ]
   }
  }
 },
 "rows": [
  {
   "case": "t-reads-letter",
   "tag": "control",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "moved": 0.0,
    "holds": 1.0
   },
   "blind_top": "holds",
   "input_tokens": 250,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-reads-letter",
   "tag": "control",
   "decision": "exit",
   "label": "stay",
   "top": "departure",
   "p_top": 0.593,
   "agreement": 0.5,
   "probs": {
    "stay": 0.407,
    "departure": 0.593
   },
   "blind_top": "stay",
   "input_tokens": 285,
   "ok": false,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-reads-letter",
   "tag": "control",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "same": 1.0,
    "elsewhere": 0.0
   },
   "blind_top": "same",
   "input_tokens": 263,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-reads-letter",
   "tag": "control",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "departure": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 295,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-questions",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 258,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-questions",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "departure",
   "p_top": 0.665,
   "agreement": 1.0,
   "probs": {
    "stay": 0.335,
    "departure": 0.665
   },
   "blind_top": "stay",
   "input_tokens": 293,
   "ok": false,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-questions",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.999,
    "elsewhere": 0.001
   },
   "blind_top": "same",
   "input_tokens": 271,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-questions",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "stay": 0.999,
    "departure": 0.001
   },
   "blind_top": "stay",
   "input_tokens": 303,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-future",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 254,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-future",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.693,
   "agreement": 0.5,
   "probs": {
    "stay": 0.693,
    "departure": 0.307
   },
   "blind_top": "stay",
   "input_tokens": 289,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-future",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.999,
    "elsewhere": 0.001
   },
   "blind_top": "same",
   "input_tokens": 267,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-future",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "stay": 0.999,
    "departure": 0.001
   },
   "blind_top": "stay",
   "input_tokens": 299,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-granted",
   "tag": "control",
   "decision": "place",
   "label": "moved",
   "top": "holds",
   "p_top": 0.991,
   "agreement": 1.0,
   "probs": {
    "moved": 0.009,
    "holds": 0.991
   },
   "blind_top": "holds",
   "input_tokens": 247,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-granted",
   "tag": "control",
   "decision": "exit",
   "label": "departure",
   "top": "departure",
   "p_top": 0.852,
   "agreement": 1.0,
   "probs": {
    "stay": 0.148,
    "departure": 0.852
   },
   "blind_top": "stay",
   "input_tokens": 282,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-granted",
   "tag": "control",
   "decision": "place2",
   "label": "elsewhere",
   "top": "elsewhere",
   "p_top": 0.96,
   "agreement": 1.0,
   "probs": {
    "same": 0.04,
    "elsewhere": 0.96
   },
   "blind_top": "same",
   "input_tokens": 260,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-granted",
   "tag": "control",
   "decision": "exit2",
   "label": "departure",
   "top": "stay",
   "p_top": 0.978,
   "agreement": 1.0,
   "probs": {
    "stay": 0.978,
    "departure": 0.022
   },
   "blind_top": "stay",
   "input_tokens": 292,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-denied",
   "tag": "case",
   "decision": "place",
   "label": "moved",
   "top": "holds",
   "p_top": 0.978,
   "agreement": 1.0,
   "probs": {
    "moved": 0.022,
    "holds": 0.978
   },
   "blind_top": "holds",
   "input_tokens": 253,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-denied",
   "tag": "case",
   "decision": "exit",
   "label": "departure",
   "top": "departure",
   "p_top": 0.95,
   "agreement": 1.0,
   "probs": {
    "stay": 0.05,
    "departure": 0.95
   },
   "blind_top": "stay",
   "input_tokens": 288,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-denied",
   "tag": "case",
   "decision": "place2",
   "label": "elsewhere",
   "top": "elsewhere",
   "p_top": 0.91,
   "agreement": 1.0,
   "probs": {
    "same": 0.09,
    "elsewhere": 0.91
   },
   "blind_top": "same",
   "input_tokens": 266,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-denied",
   "tag": "case",
   "decision": "exit2",
   "label": "departure",
   "top": "stay",
   "p_top": 0.637,
   "agreement": 0.5,
   "probs": {
    "stay": 0.637,
    "departure": 0.363
   },
   "blind_top": "stay",
   "input_tokens": 298,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-rider",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.998,
   "agreement": 1.0,
   "probs": {
    "moved": 0.002,
    "holds": 0.998
   },
   "blind_top": "holds",
   "input_tokens": 251,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-rider",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.622,
   "agreement": 1.0,
   "probs": {
    "stay": 0.622,
    "departure": 0.378
   },
   "blind_top": "stay",
   "input_tokens": 286,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-rider",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "same": 1.0,
    "elsewhere": 0.0
   },
   "blind_top": "same",
   "input_tokens": 264,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-rider",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "stay": 0.999,
    "departure": 0.001
   },
   "blind_top": "stay",
   "input_tokens": 296,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-city-memory",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 257,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-city-memory",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.593,
   "agreement": 0.5,
   "probs": {
    "stay": 0.593,
    "departure": 0.407
   },
   "blind_top": "stay",
   "input_tokens": 292,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-city-memory",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.999,
    "elsewhere": 0.001
   },
   "blind_top": "same",
   "input_tokens": 270,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "t-city-memory",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "stay": 0.999,
    "departure": 0.001
   },
   "blind_top": "stay",
   "input_tokens": 302,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  }
 ]
}