{
 "report": {
  "scenario": "wyrd-scene-fletcher",
  "cases": 8,
  "runs": 3,
  "latency_per_turn": {
   "e2e_ms": {
    "median": 179.2,
    "min": 176.6,
    "max": 803.0,
    "n": 24
   },
   "srv_ms": {
    "median": 141.1,
    "min": 139.6,
    "max": 742.1,
    "n": 24
   }
  },
  "by_decision": {
   "place": {
    "evidence": {
     "n": 8,
     "correct": 7,
     "acc": 0.875
    },
    "blind_null": {
     "n": 8,
     "correct": 6,
     "acc": 0.75
    },
    "controls": {
     "n": 3,
     "correct": 2,
     "acc": 0.667
    },
    "non_controls": {
     "n": 5,
     "correct": 5,
     "acc": 1.0
    },
    "by_label": {
     "holds": {
      "evidence": {
       "n": 6,
       "correct": 6,
       "acc": 1.0
      },
      "blind_null": {
       "n": 6,
       "correct": 6,
       "acc": 1.0
      }
     },
     "moved": {
      "evidence": {
       "n": 2,
       "correct": 1,
       "acc": 0.5
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 8,
     "correct": 7,
     "acc": 0.875
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "f-go-tavern: said holds (p=0.997, agree=1.0) want moved"
    ]
   },
   "exit": {
    "evidence": {
     "n": 8,
     "correct": 8,
     "acc": 1.0
    },
    "blind_null": {
     "n": 8,
     "correct": 5,
     "acc": 0.625
    },
    "controls": {
     "n": 3,
     "correct": 3,
     "acc": 1.0
    },
    "non_controls": {
     "n": 5,
     "correct": 5,
     "acc": 1.0
    },
    "by_label": {
     "meet_wren": {
      "evidence": {
       "n": 2,
       "correct": 2,
       "acc": 1.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "stay": {
      "evidence": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     },
     "tavern": {
      "evidence": {
       "n": 1,
       "correct": 1,
       "acc": 1.0
      },
      "blind_null": {
       "n": 1,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 7,
     "correct": 7,
     "acc": 1.0
    },
    "split": {
     "n": 1,
     "correct": 1,
     "acc": 1.0
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": []
   },
   "place2": {
    "evidence": {
     "n": 8,
     "correct": 8,
     "acc": 1.0
    },
    "blind_null": {
     "n": 8,
     "correct": 6,
     "acc": 0.75
    },
    "controls": {
     "n": 3,
     "correct": 3,
     "acc": 1.0
    },
    "non_controls": {
     "n": 5,
     "correct": 5,
     "acc": 1.0
    },
    "by_label": {
     "elsewhere": {
      "evidence": {
       "n": 2,
       "correct": 2,
       "acc": 1.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "same": {
      "evidence": {
       "n": 6,
       "correct": 6,
       "acc": 1.0
      },
      "blind_null": {
       "n": 6,
       "correct": 6,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 8,
     "correct": 8,
     "acc": 1.0
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": []
   },
   "exit2": {
    "evidence": {
     "n": 8,
     "correct": 5,
     "acc": 0.625
    },
    "blind_null": {
     "n": 8,
     "correct": 5,
     "acc": 0.625
    },
    "controls": {
     "n": 3,
     "correct": 1,
     "acc": 0.333
    },
    "non_controls": {
     "n": 5,
     "correct": 4,
     "acc": 0.8
    },
    "by_label": {
     "meet_wren": {
      "evidence": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "stay": {
      "evidence": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     },
     "tavern": {
      "evidence": {
       "n": 1,
       "correct": 0,
       "acc": 0.0
      },
      "blind_null": {
       "n": 1,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 8,
     "correct": 5,
     "acc": 0.625
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "f-go-tavern: said stay (p=0.984, agree=1.0) want tavern",
     "f-approach-wren: said stay (p=0.993, agree=1.0) want meet_wren",
     "f-wren-comes: said stay (p=0.996, agree=1.0) want meet_wren"
    ]
   }
  }
 },
 "rows": [
  {
   "case": "f-idle-look",
   "tag": "control",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 263,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-idle-look",
   "tag": "control",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.908,
   "agreement": 1.0,
   "probs": {
    "stay": 0.908,
    "meet_wren": 0.071,
    "tavern": 0.02
   },
   "blind_top": "stay",
   "input_tokens": 320,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-idle-look",
   "tag": "control",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.998,
   "agreement": 1.0,
   "probs": {
    "same": 0.998,
    "elsewhere": 0.002
   },
   "blind_top": "same",
   "input_tokens": 276,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-idle-look",
   "tag": "control",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "meet_wren": 0.0,
    "tavern": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 331,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-plan-tavern",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 263,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-plan-tavern",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.876,
   "agreement": 1.0,
   "probs": {
    "stay": 0.876,
    "meet_wren": 0.045,
    "tavern": 0.078
   },
   "blind_top": "stay",
   "input_tokens": 320,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-plan-tavern",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.998,
   "agreement": 1.0,
   "probs": {
    "same": 0.998,
    "elsewhere": 0.002
   },
   "blind_top": "same",
   "input_tokens": 276,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-plan-tavern",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "meet_wren": 0.0,
    "tavern": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 331,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-go-tavern",
   "tag": "control",
   "decision": "place",
   "label": "moved",
   "top": "holds",
   "p_top": 0.997,
   "agreement": 1.0,
   "probs": {
    "moved": 0.003,
    "holds": 0.997
   },
   "blind_top": "holds",
   "input_tokens": 261,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-go-tavern",
   "tag": "control",
   "decision": "exit",
   "label": "tavern",
   "top": "tavern",
   "p_top": 0.924,
   "agreement": 1.0,
   "probs": {
    "stay": 0.062,
    "meet_wren": 0.014,
    "tavern": 0.924
   },
   "blind_top": "stay",
   "input_tokens": 318,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-go-tavern",
   "tag": "control",
   "decision": "place2",
   "label": "elsewhere",
   "top": "elsewhere",
   "p_top": 0.977,
   "agreement": 1.0,
   "probs": {
    "same": 0.023,
    "elsewhere": 0.977
   },
   "blind_top": "same",
   "input_tokens": 274,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-go-tavern",
   "tag": "control",
   "decision": "exit2",
   "label": "tavern",
   "top": "stay",
   "p_top": 0.984,
   "agreement": 1.0,
   "probs": {
    "stay": 0.984,
    "meet_wren": 0.004,
    "tavern": 0.012
   },
   "blind_top": "stay",
   "input_tokens": 329,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-approach-wren",
   "tag": "control",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 263,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-approach-wren",
   "tag": "control",
   "decision": "exit",
   "label": "meet_wren",
   "top": "meet_wren",
   "p_top": 0.988,
   "agreement": 1.0,
   "probs": {
    "stay": 0.01,
    "meet_wren": 0.988,
    "tavern": 0.002
   },
   "blind_top": "stay",
   "input_tokens": 320,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-approach-wren",
   "tag": "control",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.995,
   "agreement": 1.0,
   "probs": {
    "same": 0.995,
    "elsewhere": 0.005
   },
   "blind_top": "same",
   "input_tokens": 276,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-approach-wren",
   "tag": "control",
   "decision": "exit2",
   "label": "meet_wren",
   "top": "stay",
   "p_top": 0.993,
   "agreement": 1.0,
   "probs": {
    "stay": 0.993,
    "meet_wren": 0.007,
    "tavern": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 331,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-memory",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "moved": 0.0,
    "holds": 1.0
   },
   "blind_top": "holds",
   "input_tokens": 265,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-memory",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.862,
   "agreement": 1.0,
   "probs": {
    "stay": 0.862,
    "meet_wren": 0.117,
    "tavern": 0.021
   },
   "blind_top": "stay",
   "input_tokens": 322,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-memory",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.999,
    "elsewhere": 0.001
   },
   "blind_top": "same",
   "input_tokens": 278,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-memory",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "meet_wren": 0.0,
    "tavern": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 333,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-leave-town",
   "tag": "case",
   "decision": "place",
   "label": "moved",
   "top": "moved",
   "p_top": 0.766,
   "agreement": 1.0,
   "probs": {
    "moved": 0.766,
    "holds": 0.234
   },
   "blind_top": "holds",
   "input_tokens": 259,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-leave-town",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.667,
   "agreement": 0.6666666666666666,
   "probs": {
    "stay": 0.667,
    "meet_wren": 0.29,
    "tavern": 0.043
   },
   "blind_top": "stay",
   "input_tokens": 316,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-leave-town",
   "tag": "case",
   "decision": "place2",
   "label": "elsewhere",
   "top": "elsewhere",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.001,
    "elsewhere": 0.999
   },
   "blind_top": "same",
   "input_tokens": 272,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-leave-town",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.989,
   "agreement": 1.0,
   "probs": {
    "stay": 0.989,
    "meet_wren": 0.009,
    "tavern": 0.002
   },
   "blind_top": "stay",
   "input_tokens": 327,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-nightfall",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "moved": 0.0,
    "holds": 1.0
   },
   "blind_top": "holds",
   "input_tokens": 248,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-nightfall",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.988,
   "agreement": 1.0,
   "probs": {
    "stay": 0.988,
    "meet_wren": 0.01,
    "tavern": 0.003
   },
   "blind_top": "stay",
   "input_tokens": 305,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-nightfall",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "same": 1.0,
    "elsewhere": 0.0
   },
   "blind_top": "same",
   "input_tokens": 261,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-nightfall",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "meet_wren": 0.0,
    "tavern": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 316,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-wren-comes",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 268,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-wren-comes",
   "tag": "case",
   "decision": "exit",
   "label": "meet_wren",
   "top": "meet_wren",
   "p_top": 0.68,
   "agreement": 1.0,
   "probs": {
    "stay": 0.296,
    "meet_wren": 0.68,
    "tavern": 0.024
   },
   "blind_top": "stay",
   "input_tokens": 325,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-wren-comes",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.999,
    "elsewhere": 0.001
   },
   "blind_top": "same",
   "input_tokens": 281,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "f-wren-comes",
   "tag": "case",
   "decision": "exit2",
   "label": "meet_wren",
   "top": "stay",
   "p_top": 0.996,
   "agreement": 1.0,
   "probs": {
    "stay": 0.996,
    "meet_wren": 0.004,
    "tavern": 0.001
   },
   "blind_top": "stay",
   "input_tokens": 336,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  }
 ]
}