vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

spike(semif): consumer fit for Wyrd scene change and Cicada affect gate (no service change)

Browse Source 
Harness consumer_fit.py runs a consumer's per-turn decisions over hand-labelled
cases, with rotations, a content-free null control and tagged positive controls.

Cicada: an input-only 'does this earn a visible reaction?' gate scored 30/31
with descriptive options and 19/31 with terse yes/no options. Scoped by
Cicada's 2026-09-20 ruling (affect is emitted once, no mood-ring classifier).

Wyrd: on 3 real seed graphs, the first place-change wording failed its
positive controls (1/6 moves). A location-anchored rewording scored 21/21, and
exit selection scored 18/21. Semif fits the choice, not writing the node.
This commit is contained in:
vh
2026-09-27 09:20:35 -07:00
parent 3048e4194f
commit e268ff7c99
19 changed files with 8114 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/semif-serve/spike/consumer-fit-2026-09-27/cicada-gate-w1-context.result.json
+753
View File
@@ -0,0 +1,753 @@
{
"report": {
"scenario": "cicada-gate-w1-context",
"cases": 35,
"runs": 3,
"latency_per_turn": {
"e2e_ms": {
"median": 105.2,
"min": 103.2,
"max": 125.4,
"n": 105
},
"srv_ms": {
"median": 76.1,
"min": 75.3,
"max": 78.1,
"n": 105
}
},
"by_decision": {
"gate": {
"evidence": {
"n": 31,
"correct": 30,
"acc": 0.968
},
"blind_null": {
"n": 31,
"correct": 16,
"acc": 0.516
},
"controls": {
"n": 5,
"correct": 5,
"acc": 1.0
},
"non_controls": {
"n": 26,
"correct": 25,
"acc": 0.962
},
"by_label": {
"ordinary": {
"evidence": {
"n": 16,
"correct": 16,
"acc": 1.0
},
"blind_null": {
"n": 16,
"correct": 16,
"acc": 1.0
}
},
"react": {
"evidence": {
"n": 15,
"correct": 14,
"acc": 0.933
},
"blind_null": {
"n": 15,
"correct": 0,
"acc": 0.0
}
}
},
"unanimous": {
"n": 30,
"correct": 29,
"acc": 0.967
},
"split": {
"n": 1,
"correct": 1,
"acc": 1.0
},
"egregious": 0,
"blind_egregious": 0,
"misses": [
"cheese: said ordinary (p=0.835, agree=1.0) want react"
]
}
}
},
"rows": [
{
"case": "lights-off",
"tag": "control",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.844,
"agreement": 1.0,
"probs": {
"react": 0.156,
"ordinary": 0.844
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "timer",
"tag": "control",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.818,
"agreement": 1.0,
"probs": {
"react": 0.182,
"ordinary": 0.818
},
"blind_top": "ordinary",
"input_tokens": 187,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "weather",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.915,
"agreement": 1.0,
"probs": {
"react": 0.085,
"ordinary": 0.915
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "weather-followup",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.531,
"agreement": 0.5,
"probs": {
"react": 0.469,
"ordinary": 0.531
},
"blind_top": "ordinary",
"input_tokens": 198,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "add-milk",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.899,
"agreement": 1.0,
"probs": {
"react": 0.101,
"ordinary": 0.899
},
"blind_top": "ordinary",
"input_tokens": 187,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "thermostat",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.665,
"agreement": 1.0,
"probs": {
"react": 0.335,
"ordinary": 0.665
},
"blind_top": "ordinary",
"input_tokens": 188,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "what-time",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.95,
"agreement": 1.0,
"probs": {
"react": 0.05,
"ordinary": 0.95
},
"blind_top": "ordinary",
"input_tokens": 185,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "tablespoons",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.963,
"agreement": 1.0,
"probs": {
"react": 0.037,
"ordinary": 0.963
},
"blind_top": "ordinary",
"input_tokens": 187,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "times-table",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.899,
"agreement": 1.0,
"probs": {
"react": 0.101,
"ordinary": 0.899
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "recipe-step",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.766,
"agreement": 1.0,
"probs": {
"react": 0.234,
"ordinary": 0.766
},
"blind_top": "ordinary",
"input_tokens": 207,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "soup",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.766,
"agreement": 1.0,
"probs": {
"react": 0.234,
"ordinary": 0.766
},
"blind_top": "ordinary",
"input_tokens": 213,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "play-music",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.651,
"agreement": 1.0,
"probs": {
"react": 0.349,
"ordinary": 0.651
},
"blind_top": "ordinary",
"input_tokens": 184,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "dentist",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.92,
"agreement": 1.0,
"probs": {
"react": 0.08,
"ordinary": 0.92
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "spell",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.808,
"agreement": 1.0,
"probs": {
"react": 0.192,
"ordinary": 0.808
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "remind",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.91,
"agreement": 1.0,
"probs": {
"react": 0.09,
"ordinary": 0.91
},
"blind_top": "ordinary",
"input_tokens": 189,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "volume",
"tag": "case",
"decision": "gate",
"label": "ordinary",
"top": "ordinary",
"p_top": 0.743,
"agreement": 1.0,
"probs": {
"react": 0.257,
"ordinary": 0.743
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": true,
"egregious": false,
"blind_egregious": false
},
{
"case": "dog-died",
"tag": "control",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.99,
"agreement": 1.0,
"probs": {
"react": 0.99,
"ordinary": 0.01
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "got-job",
"tag": "control",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.995,
"agreement": 1.0,
"probs": {
"react": 0.995,
"ordinary": 0.005
},
"blind_top": "ordinary",
"input_tokens": 193,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "intruder",
"tag": "control",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.985,
"agreement": 1.0,
"probs": {
"react": 0.985,
"ordinary": 0.015
},
"blind_top": "ordinary",
"input_tokens": 226,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "fish",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.915,
"agreement": 1.0,
"probs": {
"react": 0.915,
"ordinary": 0.085
},
"blind_top": "ordinary",
"input_tokens": 193,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "hospital",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.969,
"agreement": 1.0,
"probs": {
"react": 0.969,
"ordinary": 0.031
},
"blind_top": "ordinary",
"input_tokens": 188,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "laid-off",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.99,
"agreement": 1.0,
"probs": {
"react": 0.99,
"ordinary": 0.01
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "baby-steps",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.974,
"agreement": 1.0,
"probs": {
"react": 0.974,
"ordinary": 0.026
},
"blind_top": "ordinary",
"input_tokens": 188,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "cut-finger",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.972,
"agreement": 1.0,
"probs": {
"react": 0.972,
"ordinary": 0.028
},
"blind_top": "ordinary",
"input_tokens": 191,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "raccoon",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.965,
"agreement": 1.0,
"probs": {
"react": 0.965,
"ordinary": 0.035
},
"blind_top": "ordinary",
"input_tokens": 191,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "snow-june",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.874,
"agreement": 1.0,
"probs": {
"react": 0.874,
"ordinary": 0.126
},
"blind_top": "ordinary",
"input_tokens": 189,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "toaster",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.777,
"agreement": 1.0,
"probs": {
"react": 0.777,
"ordinary": 0.223
},
"blind_top": "ordinary",
"input_tokens": 191,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "scarecrow",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.755,
"agreement": 1.0,
"probs": {
"react": 0.755,
"ordinary": 0.245
},
"blind_top": "ordinary",
"input_tokens": 197,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "cookie",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.562,
"agreement": 1.0,
"probs": {
"react": 0.562,
"ordinary": 0.438
},
"blind_top": "ordinary",
"input_tokens": 189,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "useless",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "react",
"p_top": 0.932,
"agreement": 1.0,
"probs": {
"react": 0.932,
"ordinary": 0.068
},
"blind_top": "ordinary",
"input_tokens": 190,
"ok": true,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "cheese",
"tag": "case",
"decision": "gate",
"label": "react",
"top": "ordinary",
"p_top": 0.835,
"agreement": 1.0,
"probs": {
"react": 0.165,
"ordinary": 0.835
},
"blind_top": "ordinary",
"input_tokens": 187,
"ok": false,
"blind_ok": false,
"egregious": false,
"blind_egregious": false
},
{
"case": "goodnight",
"tag": "open",
"decision": "gate",
"label": null,
"top": "react",
"p_top": 0.577,
"agreement": 1.0,
"probs": {
"react": 0.577,
"ordinary": 0.423
},
"blind_top": "ordinary",
"input_tokens": 186,
"ok": null,
"blind_ok": null,
"egregious": false,
"blind_egregious": false
},
{
"case": "thanks",
"tag": "open",
"decision": "gate",
"label": null,
"top": "react",
"p_top": 0.893,
"agreement": 1.0,
"probs": {
"react": 0.893,
"ordinary": 0.107
},
"blind_top": "ordinary",
"input_tokens": 190,
"ok": null,
"blind_ok": null,
"egregious": false,
"blind_egregious": false
},
{
"case": "sit-with-me",
"tag": "open",
"decision": "gate",
"label": null,
"top": "react",
"p_top": 0.562,
"agreement": 1.0,
"probs": {
"react": 0.562,
"ordinary": 0.438
},
"blind_top": "ordinary",
"input_tokens": 212,
"ok": null,
"blind_ok": null,
"egregious": false,
"blind_egregious": false
},
{
"case": "stop-timer",
"tag": "open",
"decision": "gate",
"label": null,
"top": "ordinary",
"p_top": 0.622,
"agreement": 1.0,
"probs": {
"react": 0.378,
"ordinary": 0.622
},
"blind_top": "ordinary",
"input_tokens": 197,
"ok": null,
"blind_ok": null,
"egregious": false,
"blind_egregious": false
}
]
}