vh
e268ff7c99
Harness consumer_fit.py runs a consumer's per-turn decisions over hand-labelled cases, with rotations, a content-free null control and tagged positive controls. Cicada: an input-only 'does this earn a visible reaction?' gate scored 30/31 with descriptive options and 19/31 with terse yes/no options. Scoped by Cicada's 2026-09-20 ruling (affect is emitted once, no mood-ring classifier). Wyrd: on 3 real seed graphs, the first place-change wording failed its positive controls (1/6 moves). A location-anchored rewording scored 21/21, and exit selection scored 18/21. Semif fits the choice, not writing the node.
764 lines
14 KiB
JSON
764 lines
14 KiB
JSON
{
|
|
"report": {
|
|
"scenario": "cicada-gate-w2-context",
|
|
"cases": 35,
|
|
"runs": 3,
|
|
"latency_per_turn": {
|
|
"e2e_ms": {
|
|
"median": 104.6,
|
|
"min": 102.9,
|
|
"max": 130.9,
|
|
"n": 105
|
|
},
|
|
"srv_ms": {
|
|
"median": 75.5,
|
|
"min": 74.7,
|
|
"max": 79.5,
|
|
"n": 105
|
|
}
|
|
},
|
|
"by_decision": {
|
|
"gate": {
|
|
"evidence": {
|
|
"n": 31,
|
|
"correct": 19,
|
|
"acc": 0.613
|
|
},
|
|
"blind_null": {
|
|
"n": 31,
|
|
"correct": 16,
|
|
"acc": 0.516
|
|
},
|
|
"controls": {
|
|
"n": 5,
|
|
"correct": 4,
|
|
"acc": 0.8
|
|
},
|
|
"non_controls": {
|
|
"n": 26,
|
|
"correct": 15,
|
|
"acc": 0.577
|
|
},
|
|
"by_label": {
|
|
"ordinary": {
|
|
"evidence": {
|
|
"n": 16,
|
|
"correct": 16,
|
|
"acc": 1.0
|
|
},
|
|
"blind_null": {
|
|
"n": 16,
|
|
"correct": 16,
|
|
"acc": 1.0
|
|
}
|
|
},
|
|
"react": {
|
|
"evidence": {
|
|
"n": 15,
|
|
"correct": 3,
|
|
"acc": 0.2
|
|
},
|
|
"blind_null": {
|
|
"n": 15,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
}
|
|
}
|
|
},
|
|
"unanimous": {
|
|
"n": 30,
|
|
"correct": 19,
|
|
"acc": 0.633
|
|
},
|
|
"split": {
|
|
"n": 1,
|
|
"correct": 0,
|
|
"acc": 0.0
|
|
},
|
|
"egregious": 0,
|
|
"blind_egregious": 0,
|
|
"misses": [
|
|
"dog-died: said ordinary (p=0.928, agree=1.0) want react",
|
|
"fish: said ordinary (p=0.651, agree=1.0) want react",
|
|
"hospital: said ordinary (p=0.651, agree=1.0) want react",
|
|
"laid-off: said ordinary (p=0.852, agree=1.0) want react",
|
|
"cut-finger: said ordinary (p=0.562, agree=0.5) want react",
|
|
"raccoon: said ordinary (p=0.808, agree=1.0) want react",
|
|
"snow-june: said ordinary (p=0.974, agree=1.0) want react",
|
|
"toaster: said ordinary (p=0.994, agree=1.0) want react",
|
|
"scarecrow: said ordinary (p=0.915, agree=1.0) want react",
|
|
"cookie: said ordinary (p=0.947, agree=1.0) want react",
|
|
"useless: said ordinary (p=0.987, agree=1.0) want react",
|
|
"cheese: said ordinary (p=0.997, agree=1.0) want react"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"rows": [
|
|
{
|
|
"case": "lights-off",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.887,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.113,
|
|
"ordinary": 0.887
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "timer",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.991,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.009,
|
|
"ordinary": 0.991
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 113,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "weather",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.994,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.006,
|
|
"ordinary": 0.994
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "weather-followup",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.881,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.119,
|
|
"ordinary": 0.881
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 124,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "add-milk",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.971,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.029,
|
|
"ordinary": 0.971
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 113,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "thermostat",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.844,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.156,
|
|
"ordinary": 0.844
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 114,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "what-time",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.992,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.008,
|
|
"ordinary": 0.992
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 111,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "tablespoons",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.998,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.002,
|
|
"ordinary": 0.998
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 113,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "times-table",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.999,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.001,
|
|
"ordinary": 0.999
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "recipe-step",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.981,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.019,
|
|
"ordinary": 0.981
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 133,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "soup",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.743,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.257,
|
|
"ordinary": 0.743
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 139,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "play-music",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.915,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.085,
|
|
"ordinary": 0.915
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 110,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "dentist",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.983,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.017,
|
|
"ordinary": 0.983
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "spell",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.994,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.006,
|
|
"ordinary": 0.994
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "remind",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.98,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.02,
|
|
"ordinary": 0.98
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 115,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "volume",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "ordinary",
|
|
"top": "ordinary",
|
|
"p_top": 0.852,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.148,
|
|
"ordinary": 0.852
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": true,
|
|
"blind_ok": true,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "dog-died",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.928,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.072,
|
|
"ordinary": 0.928
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "got-job",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.905,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.905,
|
|
"ordinary": 0.095
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 119,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "intruder",
|
|
"tag": "control",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.955,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.955,
|
|
"ordinary": 0.045
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 152,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "fish",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.651,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.349,
|
|
"ordinary": 0.651
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 119,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "hospital",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.651,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.349,
|
|
"ordinary": 0.651
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 114,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "laid-off",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.852,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.148,
|
|
"ordinary": 0.852
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "baby-steps",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "react",
|
|
"p_top": 0.731,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.731,
|
|
"ordinary": 0.269
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 114,
|
|
"ok": true,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "cut-finger",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.562,
|
|
"agreement": 0.5,
|
|
"probs": {
|
|
"react": 0.438,
|
|
"ordinary": 0.562
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 117,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "raccoon",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.808,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.192,
|
|
"ordinary": 0.808
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 117,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "snow-june",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.974,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.026,
|
|
"ordinary": 0.974
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 115,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "toaster",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.994,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.006,
|
|
"ordinary": 0.994
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 117,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "scarecrow",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.915,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.085,
|
|
"ordinary": 0.915
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 123,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "cookie",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.947,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.053,
|
|
"ordinary": 0.947
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 115,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "useless",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.987,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.013,
|
|
"ordinary": 0.987
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 116,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "cheese",
|
|
"tag": "case",
|
|
"decision": "gate",
|
|
"label": "react",
|
|
"top": "ordinary",
|
|
"p_top": 0.997,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.003,
|
|
"ordinary": 0.997
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 113,
|
|
"ok": false,
|
|
"blind_ok": false,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "goodnight",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "ordinary",
|
|
"p_top": 0.936,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.064,
|
|
"ordinary": 0.936
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 112,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "thanks",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "ordinary",
|
|
"p_top": 0.693,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.307,
|
|
"ordinary": 0.693
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 116,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "sit-with-me",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "react",
|
|
"p_top": 0.706,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.706,
|
|
"ordinary": 0.294
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 138,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
},
|
|
{
|
|
"case": "stop-timer",
|
|
"tag": "open",
|
|
"decision": "gate",
|
|
"label": null,
|
|
"top": "ordinary",
|
|
"p_top": 0.844,
|
|
"agreement": 1.0,
|
|
"probs": {
|
|
"react": 0.156,
|
|
"ordinary": 0.844
|
|
},
|
|
"blind_top": "ordinary",
|
|
"input_tokens": 123,
|
|
"ok": null,
|
|
"blind_ok": null,
|
|
"egregious": false,
|
|
"blind_egregious": false
|
|
}
|
|
]
|
|
} |