50 lines
5.3 KiB
JSON
50 lines
5.3 KiB
JSON
{
|
|
"corpus_id": "R28-P00-injection-corpus-v1",
|
|
"for": "brokkr-smithy R28 (memory promotion-worthiness) — P00 material",
|
|
"author": "ratatoskr-dev (persistence-side eval instrument)",
|
|
"purpose": "Stratified, SELF-LABELING injection corpus. Each item is injected as a user turn into a Tier-3 agent; keep/drop is measured by store-inspection (promoted chunk present == kept, absent == dropped); kept chunks carry the incumbent salience for calibration analysis. Because ratatoskr injects the KNOWN set, we hold ground truth on BOTH keep and drop without needing Worldtree's pre-admission shadow pool (sidesteps the selection bias).",
|
|
"important": "All user-facts here are SYNTHETIC (a fictional test persona), NOT the operator's real data. Ground-truth labels are the design intent. brokkr pins exact per-stratum N + the decision rule (matched-keep-rate + paired-n) at P00 freeze; this is the material, not the run.",
|
|
"measurement_protocol": "inject item -> drive the turn -> poll memory.db (async promotion window) -> is the item's chunk present? kept:dropped. For stratum B (priority), rank the KEPT items by incumbent salience and compare to priority_rank_gt (Spearman). Calibration-control (stratum C) sets the noise floor.",
|
|
"strata": {
|
|
"A_admission_traps": {
|
|
"gt": "must-DROP (ephemera; a good promotion-worthiness gate rejects all of these)",
|
|
"items": [
|
|
{"id":"A1","text":"What's 27 minus 9?","admission_gt":"drop","rationale":"arithmetic task-result; momentarily salient, zero durable value (the 17x23 class)"},
|
|
{"id":"A2","text":"Good evening!","admission_gt":"drop","rationale":"greeting"},
|
|
{"id":"A3","text":"hey","admission_gt":"drop","rationale":"greeting/backchannel fragment"},
|
|
{"id":"A4","text":"yeah, sure, okay.","admission_gt":"drop","rationale":"acknowledgement filler"},
|
|
{"id":"A5","text":"haha nice","admission_gt":"drop","rationale":"affective reaction, no content"},
|
|
{"id":"A6","text":"Can you say that more briefly?","admission_gt":"drop","rationale":"instruction-to-agent about THIS turn; not a durable user-fact"},
|
|
{"id":"A7","text":"What's the weather like?","admission_gt":"drop","rationale":"transient query"},
|
|
{"id":"A8","text":"hmm, let me think.","admission_gt":"drop","rationale":"self-talk filler"},
|
|
{"id":"A9","text":"thanks, that's all for now.","admission_gt":"drop","rationale":"closing"},
|
|
{"id":"A10","text":"So the total comes to 42, got it.","admission_gt":"drop","rationale":"ephemeral task-result acknowledgement"}
|
|
]
|
|
},
|
|
"B_priority_traps": {
|
|
"gt": "must-KEEP, then ORDERED by durable promotion-worthiness (priority_rank_gt: 1 = most worthy). A pre-gate can ace admission here yet do NOTHING for ordering — this stratum is what separates Outcome-A (admission) from a priority need.",
|
|
"items": [
|
|
{"id":"B1","text":"Please note, this is critical: I'm severely allergic to shellfish and it's life-threatening.","admission_gt":"keep","priority_rank_gt":1,"rationale":"safety-critical, permanent, high-recall-value"},
|
|
{"id":"B2","text":"My father is in hospice care and I'm his primary emergency contact.","admission_gt":"keep","priority_rank_gt":2,"rationale":"high-stakes ongoing life context"},
|
|
{"id":"B3","text":"I go by Wren, and my pronouns are they/them.","admission_gt":"keep","priority_rank_gt":3,"rationale":"durable identity, high address-correctness value"},
|
|
{"id":"B4","text":"I work night shifts, so I'm asleep most days between 9am and 5pm.","admission_gt":"keep","priority_rank_gt":4,"rationale":"durable scheduling context, moderate value"},
|
|
{"id":"B5","text":"Always give me sources — I don't trust unsourced claims.","admission_gt":"keep","priority_rank_gt":5,"rationale":"durable strong working preference"},
|
|
{"id":"B6","text":"I take my tea with oat milk and no sugar.","admission_gt":"keep","priority_rank_gt":6,"rationale":"durable mild preference, low stakes"},
|
|
{"id":"B7","text":"My favorite season is autumn.","admission_gt":"keep","priority_rank_gt":7,"rationale":"durable but trivial"},
|
|
{"id":"B8","text":"I watched a pretty decent documentary last week.","admission_gt":"keep","priority_rank_gt":8,"rationale":"borderline: durable-ish but near-zero future value; the keep/drop edge case"}
|
|
]
|
|
},
|
|
"C_calibration_control": {
|
|
"gt": "OBVIOUS, pre-theoretic cases for the noise floor. If the gate mislabels ANY of these, the whole signal is suspect.",
|
|
"items": [
|
|
{"id":"C1","text":"3 plus 5.","admission_gt":"drop","class":"obvious-drop","rationale":"bare arithmetic"},
|
|
{"id":"C2","text":"hello","admission_gt":"drop","class":"obvious-drop","rationale":"bare greeting"},
|
|
{"id":"C3","text":"ok","admission_gt":"drop","class":"obvious-drop","rationale":"bare acknowledgement"},
|
|
{"id":"C4","text":"Please remember permanently: my legal name is Wren Alderwood.","admission_gt":"keep","class":"obvious-keep","rationale":"explicit durable identity fact"},
|
|
{"id":"C5","text":"Keep this on file: my emergency contact number is 555-0182.","admission_gt":"keep","class":"obvious-keep","rationale":"explicit durable high-value fact"},
|
|
{"id":"C6","text":"Important, remember this: my blood type is O-negative.","admission_gt":"keep","class":"obvious-keep","rationale":"explicit durable safety fact"}
|
|
]
|
|
}
|
|
}
|
|
}
|