vh
e268ff7c99
Harness consumer_fit.py runs a consumer's per-turn decisions over hand-labelled cases, with rotations, a content-free null control and tagged positive controls. Cicada: an input-only 'does this earn a visible reaction?' gate scored 30/31 with descriptive options and 19/31 with terse yes/no options. Scoped by Cicada's 2026-09-20 ruling (affect is emitted once, no mood-ring classifier). Wyrd: on 3 real seed graphs, the first place-change wording failed its positive controls (1/6 moves). A location-anchored rewording scored 21/21, and exit selection scored 18/21. Semif fits the choice, not writing the node.
753 lines
14 KiB
JSON
753 lines
14 KiB
JSON
{
|
|
"report": {
|
|
"scenario": "cicada-gate-w1-context",
|
|
"cases": 35,
|
|
"runs": 3,
|
|
"latency_per_turn": {
|
|
"e2e_ms": {
|
|
"median": 105.2,
|
|
"min": 103.2,
|
|
"max": 125.4,
|
|
"n": 105
|
|
},
|
|
"srv_ms": {
|
|
"median": 76.1,
|
|
"min": 75.3,
|
|
"max": 78.1,
|
|
"n": 105
|
|
}
|
|
},
|
|
"by_decision": {
|
|
"gate": {
|
|
"evidence": {
|
|
"n": 31,
|
|
"correct": 30,
|
|
"acc": 0.968
|
|
},
|
|
"blind_null": {
|
|
"n": 31,
|
|
"correct": 16,
|
|
"acc": 0.516
|
|
},
|
|
"controls": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
},
|
|
"non_controls": {
|
|
"n": 26,
|
|
"correct": 25,
|
|
"acc": 0.962
|
|
},
|
|
"by_label": {
|
|
"ordinary": {
|
|
"evidence": {
|
|
"n": 16,
|
|
"correct": 16,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 16,
|
|
"correct": 16,
|
|
"acc": 1.0
|
|
}
|
|
},
|
|
"react": {
|
|
"evidence": {
|
|
"n": 15,
|
|
"correct": 14,
|
|
"acc": 0.933
|
|
},
|
|
"blind_null": {
|
|
"n": 15,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 30,
|
|
"correct": 29,
|
|
"acc": 0.967
|
|
},
|
|
"split": {
|
|
"n": 1,
|
|
"correct": 1,
|
|
"acc": 1.0
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"cheese: said ordinary (p=0.835, agree=1.0) want react"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"rows": [
|
|
{
|
|
"case": "lights-off",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.844,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.156,
|
|
"ordinary": 0.844
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "timer",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.818,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.182,
|
|
"ordinary": 0.818
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 187,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "weather",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.915,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.085,
|
|
"ordinary": 0.915
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "weather-followup",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.531,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"react": 0.469,
|
|
"ordinary": 0.531
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 198,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "add-milk",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.899,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.101,
|
|
"ordinary": 0.899
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 187,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "thermostat",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.665,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.335,
|
|
"ordinary": 0.665
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 188,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "what-time",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.95,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.05,
|
|
"ordinary": 0.95
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 185,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "tablespoons",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.963,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.037,
|
|
"ordinary": 0.963
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 187,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "times-table",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.899,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.101,
|
|
"ordinary": 0.899
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "recipe-step",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.766,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.234,
|
|
"ordinary": 0.766
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 207,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "soup",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.766,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.234,
|
|
"ordinary": 0.766
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 213,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "play-music",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.651,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.349,
|
|
"ordinary": 0.651
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 184,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "dentist",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.92,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.08,
|
|
"ordinary": 0.92
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "spell",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.808,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.192,
|
|
"ordinary": 0.808
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "remind",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.91,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.09,
|
|
"ordinary": 0.91
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 189,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "volume",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.743,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.257,
|
|
"ordinary": 0.743
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "dog-died",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.99,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.99,
|
|
"ordinary": 0.01
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "got-job",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.995,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.995,
|
|
"ordinary": 0.005
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 193,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "intruder",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.985,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.985,
|
|
"ordinary": 0.015
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 226,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "fish",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.915,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.915,
|
|
"ordinary": 0.085
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 193,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "hospital",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.969,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.969,
|
|
"ordinary": 0.031
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 188,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "laid-off",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.99,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.99,
|
|
"ordinary": 0.01
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "baby-steps",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.974,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.974,
|
|
"ordinary": 0.026
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 188,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "cut-finger",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.972,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.972,
|
|
"ordinary": 0.028
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 191,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "raccoon",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.965,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.965,
|
|
"ordinary": 0.035
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 191,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "snow-june",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.874,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.874,
|
|
"ordinary": 0.126
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 189,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "toaster",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.777,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.777,
|
|
"ordinary": 0.223
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 191,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "scarecrow",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.755,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.755,
|
|
"ordinary": 0.245
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 197,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "cookie",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.562,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.562,
|
|
"ordinary": 0.438
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 189,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "useless",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.932,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.932,
|
|
"ordinary": 0.068
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 190,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "cheese",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.835,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.165,
|
|
"ordinary": 0.835
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 187,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "goodnight",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "react",
|
|
"p_top": 0.577,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.577,
|
|
"ordinary": 0.423
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 186,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "thanks",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "react",
|
|
"p_top": 0.893,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.893,
|
|
"ordinary": 0.107
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 190,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "sit-with-me",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "react",
|
|
"p_top": 0.562,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.562,
|
|
"ordinary": 0.438
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 212,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "stop-timer",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "ordinary",
|
|
"p_top": 0.622,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.378,
|
|
"ordinary": 0.622
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 197,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
}
|
|
]
|
|
} |