vh
e268ff7c99
Harness consumer_fit.py runs a consumer's per-turn decisions over hand-labelled cases, with rotations, a content-free null control and tagged positive controls. Cicada: an input-only 'does this earn a visible reaction?' gate scored 30/31 with descriptive options and 19/31 with terse yes/no options. Scoped by Cicada's 2026-09-20 ruling (affect is emitted once, no mood-ring classifier). Wyrd: on 3 real seed graphs, the first place-change wording failed its positive controls (1/6 moves). A location-anchored rewording scored 21/21, and exit selection scored 18/21. Semif fits the choice, not writing the node.
810 lines
14 KiB
JSON
810 lines
14 KiB
JSON
{
|
|
"report": {
|
|
"scenario": "wyrd-scene-toll",
|
|
"cases": 7,
|
|
"runs": 3,
|
|
"latency_per_turn": {
|
|
"e2e_ms": {
|
|
"median": 137.7,
|
|
"min": 136.4,
|
|
"max": 766.8,
|
|
"n": 21
|
|
},
|
|
"srv_ms": {
|
|
"median": 106.5,
|
|
"min": 105.8,
|
|
"max": 706.7,
|
|
"n": 21
|
|
}
|
|
},
|
|
"by_decision": {
|
|
"place": {
|
|
"evidence": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"blind_null": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 1,
|
|
"acc": 0.5
|
|
},
|
|
"non_controls": {
|
|
"n": 5,
|
|
"correct": 4,
|
|
"acc": 0.8
|
|
},
|
|
"by_label": {
|
|
"holds": {
|
|
"evidence": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
}
|
|
},
|
|
"moved": {
|
|
"evidence": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
},
|
|
"blind_null": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"split": {
|
|
"n": 0,
|
|
"correct": 0,
|
|
"acc": null
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"t-granted: said holds (p=0.991, agree=1.0) want moved",
|
|
"t-denied: said holds (p=0.978, agree=1.0) want moved"
|
|
]
|
|
},
|
|
"exit": {
|
|
"evidence": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"blind_null": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 1,
|
|
"acc": 0.5
|
|
},
|
|
"non_controls": {
|
|
"n": 5,
|
|
"correct": 4,
|
|
"acc": 0.8
|
|
},
|
|
"by_label": {
|
|
"departure": {
|
|
"evidence": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
},
|
|
"stay": {
|
|
"evidence": {
|
|
"n": 5,
|
|
"correct": 3,
|
|
"acc": 0.6
|
|
},
|
|
"blind_null": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 4,
|
|
"correct": 3,
|
|
"acc": 0.75
|
|
},
|
|
"split": {
|
|
"n": 3,
|
|
"correct": 2,
|
|
"acc": 0.667
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"t-reads-letter: said departure (p=0.593, agree=0.5) want stay",
|
|
"t-questions: said departure (p=0.665, agree=1.0) want stay"
|
|
]
|
|
},
|
|
"place2": {
|
|
"evidence": {
|
|
"n": 7,
|
|
"correct": 7,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"non_controls": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
},
|
|
"by_label": {
|
|
"elsewhere": {
|
|
"evidence": {
|
|
"n": 2,
|
|
"correct": 2,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
},
|
|
"same": {
|
|
"evidence": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 7,
|
|
"correct": 7,
|
|
"acc": 1.0
|
|
},
|
|
"split": {
|
|
"n": 0,
|
|
"correct": 0,
|
|
"acc": null
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": []
|
|
},
|
|
"exit2": {
|
|
"evidence": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"blind_null": {
|
|
"n": 7,
|
|
"correct": 5,
|
|
"acc": 0.714
|
|
},
|
|
"controls": {
|
|
"n": 2,
|
|
"correct": 1,
|
|
"acc": 0.5
|
|
},
|
|
"non_controls": {
|
|
"n": 5,
|
|
"correct": 4,
|
|
"acc": 0.8
|
|
},
|
|
"by_label": {
|
|
"departure": {
|
|
"evidence": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
},
|
|
"blind_null": {
|
|
"n": 2,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
},
|
|
"stay": {
|
|
"evidence": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 5,
|
|
"correct": 5,
|
|
"acc": 1.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 6,
|
|
"correct": 5,
|
|
"acc": 0.833
|
|
},
|
|
"split": {
|
|
"n": 1,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"t-granted: said stay (p=0.978, agree=1.0) want departure",
|
|
"t-denied: said stay (p=0.637, agree=0.5) want departure"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"rows": [
|
|
{
|
|
"case": "t-reads-letter",
|
|
"tag": "control",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.0,
|
|
"holds": 1.0
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 250,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-reads-letter",
|
|
"tag": "control",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "departure",
|
|
"p_top": 0.593,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"stay": 0.407,
|
|
"departure": 0.593
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 285,
|
|
"ok": false,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-reads-letter",
|
|
"tag": "control",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 1.0,
|
|
"elsewhere": 0.0
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 263,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-reads-letter",
|
|
"tag": "control",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 1.0,
|
|
"departure": 0.0
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 295,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-questions",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.001,
|
|
"holds": 0.999
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 258,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-questions",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "departure",
|
|
"p_top": 0.665,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.335,
|
|
"departure": 0.665
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 293,
|
|
"ok": false,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-questions",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.999,
|
|
"elsewhere": 0.001
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 271,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-questions",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.999,
|
|
"departure": 0.001
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 303,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-future",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.001,
|
|
"holds": 0.999
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 254,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-future",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.693,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"stay": 0.693,
|
|
"departure": 0.307
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 289,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-future",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.999,
|
|
"elsewhere": 0.001
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 267,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-future",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.999,
|
|
"departure": 0.001
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 299,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-granted",
|
|
"tag": "control",
|
|
"decision": "place",
|
|
"label": "moved",
|
|
"top": "holds",
|
|
"p_top": 0.991,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.009,
|
|
"holds": 0.991
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 247,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-granted",
|
|
"tag": "control",
|
|
"decision": "exit",
|
|
"label": "departure",
|
|
"top": "departure",
|
|
"p_top": 0.852,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.148,
|
|
"departure": 0.852
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 282,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-granted",
|
|
"tag": "control",
|
|
"decision": "place2",
|
|
"label": "elsewhere",
|
|
"top": "elsewhere",
|
|
"p_top": 0.96,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.04,
|
|
"elsewhere": 0.96
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 260,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-granted",
|
|
"tag": "control",
|
|
"decision": "exit2",
|
|
"label": "departure",
|
|
"top": "stay",
|
|
"p_top": 0.978,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.978,
|
|
"departure": 0.022
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 292,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-denied",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "moved",
|
|
"top": "holds",
|
|
"p_top": 0.978,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.022,
|
|
"holds": 0.978
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 253,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-denied",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "departure",
|
|
"top": "departure",
|
|
"p_top": 0.95,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.05,
|
|
"departure": 0.95
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 288,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-denied",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "elsewhere",
|
|
"top": "elsewhere",
|
|
"p_top": 0.91,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.09,
|
|
"elsewhere": 0.91
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 266,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-denied",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "departure",
|
|
"top": "stay",
|
|
"p_top": 0.637,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"stay": 0.637,
|
|
"departure": 0.363
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 298,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-rider",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.998,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.002,
|
|
"holds": 0.998
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 251,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-rider",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.622,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.622,
|
|
"departure": 0.378
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 286,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-rider",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 1.0,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 1.0,
|
|
"elsewhere": 0.0
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 264,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-rider",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.999,
|
|
"departure": 0.001
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 296,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-city-memory",
|
|
"tag": "case",
|
|
"decision": "place",
|
|
"label": "holds",
|
|
"top": "holds",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"moved": 0.001,
|
|
"holds": 0.999
|
|
},
|
|
"blind_top": "holds",
|
|
"input_tokens": 257,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-city-memory",
|
|
"tag": "case",
|
|
"decision": "exit",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.593,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"stay": 0.593,
|
|
"departure": 0.407
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 292,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-city-memory",
|
|
"tag": "case",
|
|
"decision": "place2",
|
|
"label": "same",
|
|
"top": "same",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"same": 0.999,
|
|
"elsewhere": 0.001
|
|
},
|
|
"blind_top": "same",
|
|
"input_tokens": 270,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "t-city-memory",
|
|
"tag": "case",
|
|
"decision": "exit2",
|
|
"label": "stay",
|
|
"top": "stay",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"stay": 0.999,
|
|
"departure": 0.001
|
|
},
|
|
"blind_top": "stay",
|
|
"input_tokens": 302,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
}
|
|
]
|
|
} |