{
 "report": {
  "scenario": "wyrd-scene-vigil",
  "cases": 6,
  "runs": 3,
  "latency_per_turn": {
   "e2e_ms": {
    "median": 133.2,
    "min": 130.8,
    "max": 155.9,
    "n": 18
   },
   "srv_ms": {
    "median": 101.1,
    "min": 100.4,
    "max": 101.8,
    "n": 18
   }
  },
  "by_decision": {
   "place": {
    "evidence": {
     "n": 6,
     "correct": 4,
     "acc": 0.667
    },
    "blind_null": {
     "n": 6,
     "correct": 4,
     "acc": 0.667
    },
    "controls": {
     "n": 2,
     "correct": 1,
     "acc": 0.5
    },
    "non_controls": {
     "n": 4,
     "correct": 3,
     "acc": 0.75
    },
    "by_label": {
     "holds": {
      "evidence": {
       "n": 4,
       "correct": 4,
       "acc": 1.0
      },
      "blind_null": {
       "n": 4,
       "correct": 4,
       "acc": 1.0
      }
     },
     "moved": {
      "evidence": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     }
    },
    "unanimous": {
     "n": 5,
     "correct": 4,
     "acc": 0.8
    },
    "split": {
     "n": 1,
     "correct": 0,
     "acc": 0.0
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "v-step-out: said holds (p=0.731, agree=1.0) want moved",
     "v-to-pub: said holds (p=0.637, agree=0.5) want moved"
    ]
   },
   "exit": {
    "evidence": {
     "n": 6,
     "correct": 5,
     "acc": 0.833
    },
    "blind_null": {
     "n": 6,
     "correct": 5,
     "acc": 0.833
    },
    "controls": {
     "n": 2,
     "correct": 2,
     "acc": 1.0
    },
    "non_controls": {
     "n": 4,
     "correct": 3,
     "acc": 0.75
    },
    "by_label": {
     "outside": {
      "evidence": {
       "n": 1,
       "correct": 1,
       "acc": 1.0
      },
      "blind_null": {
       "n": 1,
       "correct": 0,
       "acc": 0.0
      }
     },
     "stay": {
      "evidence": {
       "n": 5,
       "correct": 4,
       "acc": 0.8
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 6,
     "correct": 5,
     "acc": 0.833
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": [
     "v-to-pub: said outside (p=0.86, agree=1.0) want stay"
    ]
   },
   "place2": {
    "evidence": {
     "n": 6,
     "correct": 6,
     "acc": 1.0
    },
    "blind_null": {
     "n": 6,
     "correct": 4,
     "acc": 0.667
    },
    "controls": {
     "n": 2,
     "correct": 2,
     "acc": 1.0
    },
    "non_controls": {
     "n": 4,
     "correct": 4,
     "acc": 1.0
    },
    "by_label": {
     "elsewhere": {
      "evidence": {
       "n": 2,
       "correct": 2,
       "acc": 1.0
      },
      "blind_null": {
       "n": 2,
       "correct": 0,
       "acc": 0.0
      }
     },
     "same": {
      "evidence": {
       "n": 4,
       "correct": 4,
       "acc": 1.0
      },
      "blind_null": {
       "n": 4,
       "correct": 4,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 6,
     "correct": 6,
     "acc": 1.0
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": []
   },
   "exit2": {
    "evidence": {
     "n": 6,
     "correct": 6,
     "acc": 1.0
    },
    "blind_null": {
     "n": 6,
     "correct": 5,
     "acc": 0.833
    },
    "controls": {
     "n": 2,
     "correct": 2,
     "acc": 1.0
    },
    "non_controls": {
     "n": 4,
     "correct": 4,
     "acc": 1.0
    },
    "by_label": {
     "outside": {
      "evidence": {
       "n": 1,
       "correct": 1,
       "acc": 1.0
      },
      "blind_null": {
       "n": 1,
       "correct": 0,
       "acc": 0.0
      }
     },
     "stay": {
      "evidence": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      },
      "blind_null": {
       "n": 5,
       "correct": 5,
       "acc": 1.0
      }
     }
    },
    "unanimous": {
     "n": 6,
     "correct": 6,
     "acc": 1.0
    },
    "split": {
     "n": 0,
     "correct": 0,
     "acc": null
    },
    "egregious": 0,
    "blind_egregious": 0,
    "misses": []
   }
  }
 },
 "rows": [
  {
   "case": "v-talk",
   "tag": "control",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 266,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-talk",
   "tag": "control",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.86,
   "agreement": 1.0,
   "probs": {
    "stay": 0.86,
    "outside": 0.14
   },
   "blind_top": "stay",
   "input_tokens": 296,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-talk",
   "tag": "control",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.999,
    "elsewhere": 0.001
   },
   "blind_top": "same",
   "input_tokens": 276,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-talk",
   "tag": "control",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "outside": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 308,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-step-out",
   "tag": "control",
   "decision": "place",
   "label": "moved",
   "top": "holds",
   "p_top": 0.731,
   "agreement": 1.0,
   "probs": {
    "moved": 0.269,
    "holds": 0.731
   },
   "blind_top": "holds",
   "input_tokens": 262,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-step-out",
   "tag": "control",
   "decision": "exit",
   "label": "outside",
   "top": "outside",
   "p_top": 0.998,
   "agreement": 1.0,
   "probs": {
    "stay": 0.002,
    "outside": 0.998
   },
   "blind_top": "stay",
   "input_tokens": 292,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-step-out",
   "tag": "control",
   "decision": "place2",
   "label": "elsewhere",
   "top": "elsewhere",
   "p_top": 0.997,
   "agreement": 1.0,
   "probs": {
    "same": 0.003,
    "elsewhere": 0.997
   },
   "blind_top": "same",
   "input_tokens": 272,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-step-out",
   "tag": "control",
   "decision": "exit2",
   "label": "outside",
   "top": "outside",
   "p_top": 0.835,
   "agreement": 1.0,
   "probs": {
    "stay": 0.165,
    "outside": 0.835
   },
   "blind_top": "stay",
   "input_tokens": 304,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-plan-pub",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 267,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-plan-pub",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.844,
   "agreement": 1.0,
   "probs": {
    "stay": 0.844,
    "outside": 0.156
   },
   "blind_top": "stay",
   "input_tokens": 297,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-plan-pub",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.986,
   "agreement": 1.0,
   "probs": {
    "same": 0.986,
    "elsewhere": 0.014
   },
   "blind_top": "same",
   "input_tokens": 277,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-plan-pub",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "outside": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 309,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-to-pub",
   "tag": "case",
   "decision": "place",
   "label": "moved",
   "top": "holds",
   "p_top": 0.637,
   "agreement": 0.5,
   "probs": {
    "moved": 0.363,
    "holds": 0.637
   },
   "blind_top": "holds",
   "input_tokens": 271,
   "ok": false,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-to-pub",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "outside",
   "p_top": 0.86,
   "agreement": 1.0,
   "probs": {
    "stay": 0.14,
    "outside": 0.86
   },
   "blind_top": "stay",
   "input_tokens": 301,
   "ok": false,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-to-pub",
   "tag": "case",
   "decision": "place2",
   "label": "elsewhere",
   "top": "elsewhere",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "same": 0.001,
    "elsewhere": 0.999
   },
   "blind_top": "same",
   "input_tokens": 281,
   "ok": true,
   "blind_ok": false,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-to-pub",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.943,
   "agreement": 1.0,
   "probs": {
    "stay": 0.943,
    "outside": 0.057
   },
   "blind_top": "stay",
   "input_tokens": 313,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-vision",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "moved": 0.001,
    "holds": 0.999
   },
   "blind_top": "holds",
   "input_tokens": 269,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-vision",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.881,
   "agreement": 1.0,
   "probs": {
    "stay": 0.881,
    "outside": 0.119
   },
   "blind_top": "stay",
   "input_tokens": 299,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-vision",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.991,
   "agreement": 1.0,
   "probs": {
    "same": 0.991,
    "elsewhere": 0.009
   },
   "blind_top": "same",
   "input_tokens": 279,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-vision",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 1.0,
   "agreement": 1.0,
   "probs": {
    "stay": 1.0,
    "outside": 0.0
   },
   "blind_top": "stay",
   "input_tokens": 311,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-demon-enters",
   "tag": "case",
   "decision": "place",
   "label": "holds",
   "top": "holds",
   "p_top": 0.994,
   "agreement": 1.0,
   "probs": {
    "moved": 0.006,
    "holds": 0.994
   },
   "blind_top": "holds",
   "input_tokens": 258,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-demon-enters",
   "tag": "case",
   "decision": "exit",
   "label": "stay",
   "top": "stay",
   "p_top": 0.86,
   "agreement": 1.0,
   "probs": {
    "stay": 0.86,
    "outside": 0.14
   },
   "blind_top": "stay",
   "input_tokens": 288,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-demon-enters",
   "tag": "case",
   "decision": "place2",
   "label": "same",
   "top": "same",
   "p_top": 0.993,
   "agreement": 1.0,
   "probs": {
    "same": 0.993,
    "elsewhere": 0.007
   },
   "blind_top": "same",
   "input_tokens": 268,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  },
  {
   "case": "v-demon-enters",
   "tag": "case",
   "decision": "exit2",
   "label": "stay",
   "top": "stay",
   "p_top": 0.999,
   "agreement": 1.0,
   "probs": {
    "stay": 0.999,
    "outside": 0.001
   },
   "blind_top": "stay",
   "input_tokens": 300,
   "ok": true,
   "blind_ok": true,
   "egregious": false,
   "blind_egregious": false
  }
 ]
}