vh
e268ff7c99
Harness consumer_fit.py runs a consumer's per-turn decisions over hand-labelled cases, with rotations, a content-free null control and tagged positive controls. Cicada: an input-only 'does this earn a visible reaction?' gate scored 30/31 with descriptive options and 19/31 with terse yes/no options. Scoped by Cicada's 2026-09-20 ruling (affect is emitted once, no mood-ring classifier). Wyrd: on 3 real seed graphs, the first place-change wording failed its positive controls (1/6 moves). A location-anchored rewording scored 21/21, and exit selection scored 18/21. Semif fits the choice, not writing the node.
730 lines
12 KiB
JSON
730 lines
12 KiB
JSON
{
|
|
"report": {
|
|
"scenario": "wyrd-scene-vigil",
|
|
"cases": 6,
|
|
"runs": 3,
|
|
"latency_per_turn": {
|
|
"e2e_ms": {
|
|
"median": 133.2,
|
|
"min": 130.8,
|
|
"max": 155.9,
|
|
"n": 18
|
|
},
|
|
"srv_ms": {
|
|
"median": 101.1,
|
|
"min": 100.4,
|
|
"max": 101.8,
|
|
"n": 18
|
|
}
|
|
},
|
|
"by_decision": {
|
|
"place": {
|
|
"evidence": {
|
|
"n": 6,
|
|
"correct": 4,
|
|
"acc": 0.667
|
|
},
|
|
"blind_null": {
|
|
"n": 6,
|
|
"correct": 4,
|
|
"acc": 0.667
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 1,
|
|
"acc": 0.5
|
|
},
|
|
"non_controls": {
|
|
"n": 4,
|
|
"correct": 3,
|
|
"acc": 0.75
|
|
},
|
|
"by_label": {
|
|
"holds": {
|
|
"evidence": {
|
|
"n": 4,
|
|
"correct": 4,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 4,
|
|
"correct": 4,
|
|
"acc": 1.0
|
|
}
|
|
},
|
|
"moved": {
|
|
"evidence": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
},
|
|
"blind_null": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 5,
|
|
"correct": 4,
|
|
"acc": 0.8
|
|
},
|
|
"split": {
|
|
"n": 1,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"v-step-out: said holds (p=0.731, agree=1.0) want moved",
|
|
"v-to-pub: said holds (p=0.637, agree=0.5) want moved"
|
|
]
|
|
},
|
|
"exit": {
|
|
"evidence": {
|
|
"n": 6,
|
|
"correct": 5,
|
|
"acc": 0.833
|
|
},
|
|
"blind_null": {
|
|
"n": 6,
|
|
"correct": 5,
|
|
"acc": 0.833
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"non_controls": {
|
|
"n": 4,
|
|
"correct": 3,
|
|
"acc": 0.75
|
|
},
|
|
"by_label": {
|
|
"outside": {
|
|
"evidence": {
|
|
"n": 1,
|
|
"correct": 1,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 1,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
},
|
|
"stay": {
|
|
"evidence": {
|
|
"n": 5,
|
|
"correct": 4,
|
|
"acc": 0.8
|
|
},
|
|
"blind_null": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 6,
|
|
"correct": 5,
|
|
"acc": 0.833
|
|
},
|
|
"split": {
|
|
"n": 0,
|
|
"correct": 0,
|
|
"acc": null
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"v-to-pub: said outside (p=0.86, agree=1.0) want stay"
|
|
]
|
|
},
|
|
"place2": {
|
|
"evidence": {
|
|
"n": 6,
|
|
"correct": 6,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 6,
|
|
"correct": 4,
|
|
"acc": 0.667
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"non_controls": {
|
|
"n": 4,
|
|
"correct": 4,
|
|
"acc": 1.0
|
|
},
|
|
"by_label": {
|
|
"elsewhere": {
|
|
"evidence": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
},
|
|
"same": {
|
|
"evidence": {
|
|
"n": 4,
|
|
"correct": 4,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 4,
|
|
"correct": 4,
|
|
"acc": 1.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 6,
|
|
"correct": 6,
|
|
"acc": 1.0
|
|
},
|
|
"split": {
|
|
"n": 0,
|
|
"correct": 0,
|
|
"acc": null
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": []
|
|
},
|
|
"exit2": {
|
|
"evidence": {
|
|
"n": 6,
|
|
"correct": 6,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 6,
|
|
"correct": 5,
|
|
"acc": 0.833
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"non_controls": {
|
|
"n": 4,
|
|
"correct": 4,
|
|
"acc": 1.0
|
|
},
|
|
"by_label": {
|
|
"outside": {
|
|
"evidence": {
|
|
"n": 1,
|
|
"correct": 1,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 1,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
},
|
|
"stay": {
|
|
"evidence": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 6,
|
|
"correct": 6,
|
|
"acc": 1.0
|
|
},
|
|
"split": {
|
|
"n": 0,
|
|
"correct": 0,
|
|
"acc": null
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": []
|
|
}
|
|
}
|
|
},
|
|
"rows": [
|
|
{
|
|
"case": "v-talk",
|
|
"tag": "control",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.001,
|
|
"holds": 0.999
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 266,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-talk",
|
|
"tag": "control",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.86,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.86,
|
|
"outside": 0.14
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 296,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-talk",
|
|
"tag": "control",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.999,
|
|
"elsewhere": 0.001
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 276,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-talk",
|
|
"tag": "control",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 1.0,
|
|
"outside": 0.0
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 308,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-step-out",
|
|
"tag": "control",
|
|
"decision": "place",
|
|
"label": "moved",
|
|
"top": "holds",
|
|
"p_top": 0.731,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.269,
|
|
"holds": 0.731
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 262,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-step-out",
|
|
"tag": "control",
|
|
"decision": "exit",
|
|
"label": "outside",
|
|
"top": "outside",
|
|
"p_top": 0.998,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.002,
|
|
"outside": 0.998
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 292,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-step-out",
|
|
"tag": "control",
|
|
"decision": "place2",
|
|
"label": "elsewhere",
|
|
"top": "elsewhere",
|
|
"p_top": 0.997,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.003,
|
|
"elsewhere": 0.997
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 272,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-step-out",
|
|
"tag": "control",
|
|
"decision": "exit2",
|
|
"label": "outside",
|
|
"top": "outside",
|
|
"p_top": 0.835,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.165,
|
|
"outside": 0.835
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 304,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-plan-pub",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.001,
|
|
"holds": 0.999
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 267,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-plan-pub",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.844,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.844,
|
|
"outside": 0.156
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 297,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-plan-pub",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.986,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.986,
|
|
"elsewhere": 0.014
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 277,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-plan-pub",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 1.0,
|
|
"outside": 0.0
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 309,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-to-pub",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "moved",
|
|
"top": "holds",
|
|
"p_top": 0.637,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"moved": 0.363,
|
|
"holds": 0.637
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 271,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-to-pub",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "outside",
|
|
"p_top": 0.86,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.14,
|
|
"outside": 0.86
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 301,
|
|
"ok": false,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-to-pub",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "elsewhere",
|
|
"top": "elsewhere",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.001,
|
|
"elsewhere": 0.999
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 281,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-to-pub",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.943,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.943,
|
|
"outside": 0.057
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 313,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-vision",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.001,
|
|
"holds": 0.999
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 269,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-vision",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.881,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.881,
|
|
"outside": 0.119
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 299,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-vision",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.991,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.991,
|
|
"elsewhere": 0.009
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 279,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-vision",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 1.0,
|
|
"outside": 0.0
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 311,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-demon-enters",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.994,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.006,
|
|
"holds": 0.994
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 258,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-demon-enters",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.86,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.86,
|
|
"outside": 0.14
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 288,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-demon-enters",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.993,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.993,
|
|
"elsewhere": 0.007
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 268,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "v-demon-enters",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.999,
|
|
"outside": 0.001
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 300,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
}
|
|
]
|
|
} |