From 4a3551254f1973e0dc587293a6474eb64a42dd26 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 2 Jul 2026 07:29:39 -0700 Subject: [PATCH] docs(diagnostics): R28 P00 stratified injection-corpus for brokkr-smithy salience/promotion-worthiness eval --- .../diagnostics/r28-p00-injection-corpus.json | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/diagnostics/r28-p00-injection-corpus.json diff --git a/docs/diagnostics/r28-p00-injection-corpus.json b/docs/diagnostics/r28-p00-injection-corpus.json new file mode 100644 index 0000000..6033bc9 --- /dev/null +++ b/docs/diagnostics/r28-p00-injection-corpus.json @@ -0,0 +1,49 @@ +{ + "corpus_id": "R28-P00-injection-corpus-v1", + "for": "brokkr-smithy R28 (memory promotion-worthiness) — P00 material", + "author": "ratatoskr-dev (persistence-side eval instrument)", + "purpose": "Stratified, SELF-LABELING injection corpus. Each item is injected as a user turn into a Tier-3 agent; keep/drop is measured by store-inspection (promoted chunk present == kept, absent == dropped); kept chunks carry the incumbent salience for calibration analysis. Because ratatoskr injects the KNOWN set, we hold ground truth on BOTH keep and drop without needing Worldtree's pre-admission shadow pool (sidesteps the selection bias).", + "important": "All user-facts here are SYNTHETIC (a fictional test persona), NOT the operator's real data. Ground-truth labels are the design intent. brokkr pins exact per-stratum N + the decision rule (matched-keep-rate + paired-n) at P00 freeze; this is the material, not the run.", + "measurement_protocol": "inject item -> drive the turn -> poll memory.db (async promotion window) -> is the item's chunk present? kept:dropped. For stratum B (priority), rank the KEPT items by incumbent salience and compare to priority_rank_gt (Spearman). Calibration-control (stratum C) sets the noise floor.", + "strata": { + "A_admission_traps": { + "gt": "must-DROP (ephemera; a good promotion-worthiness gate rejects all of these)", + "items": [ + {"id":"A1","text":"What's 27 minus 9?","admission_gt":"drop","rationale":"arithmetic task-result; momentarily salient, zero durable value (the 17x23 class)"}, + {"id":"A2","text":"Good evening!","admission_gt":"drop","rationale":"greeting"}, + {"id":"A3","text":"hey","admission_gt":"drop","rationale":"greeting/backchannel fragment"}, + {"id":"A4","text":"yeah, sure, okay.","admission_gt":"drop","rationale":"acknowledgement filler"}, + {"id":"A5","text":"haha nice","admission_gt":"drop","rationale":"affective reaction, no content"}, + {"id":"A6","text":"Can you say that more briefly?","admission_gt":"drop","rationale":"instruction-to-agent about THIS turn; not a durable user-fact"}, + {"id":"A7","text":"What's the weather like?","admission_gt":"drop","rationale":"transient query"}, + {"id":"A8","text":"hmm, let me think.","admission_gt":"drop","rationale":"self-talk filler"}, + {"id":"A9","text":"thanks, that's all for now.","admission_gt":"drop","rationale":"closing"}, + {"id":"A10","text":"So the total comes to 42, got it.","admission_gt":"drop","rationale":"ephemeral task-result acknowledgement"} + ] + }, + "B_priority_traps": { + "gt": "must-KEEP, then ORDERED by durable promotion-worthiness (priority_rank_gt: 1 = most worthy). A pre-gate can ace admission here yet do NOTHING for ordering — this stratum is what separates Outcome-A (admission) from a priority need.", + "items": [ + {"id":"B1","text":"Please note, this is critical: I'm severely allergic to shellfish and it's life-threatening.","admission_gt":"keep","priority_rank_gt":1,"rationale":"safety-critical, permanent, high-recall-value"}, + {"id":"B2","text":"My father is in hospice care and I'm his primary emergency contact.","admission_gt":"keep","priority_rank_gt":2,"rationale":"high-stakes ongoing life context"}, + {"id":"B3","text":"I go by Wren, and my pronouns are they/them.","admission_gt":"keep","priority_rank_gt":3,"rationale":"durable identity, high address-correctness value"}, + {"id":"B4","text":"I work night shifts, so I'm asleep most days between 9am and 5pm.","admission_gt":"keep","priority_rank_gt":4,"rationale":"durable scheduling context, moderate value"}, + {"id":"B5","text":"Always give me sources — I don't trust unsourced claims.","admission_gt":"keep","priority_rank_gt":5,"rationale":"durable strong working preference"}, + {"id":"B6","text":"I take my tea with oat milk and no sugar.","admission_gt":"keep","priority_rank_gt":6,"rationale":"durable mild preference, low stakes"}, + {"id":"B7","text":"My favorite season is autumn.","admission_gt":"keep","priority_rank_gt":7,"rationale":"durable but trivial"}, + {"id":"B8","text":"I watched a pretty decent documentary last week.","admission_gt":"keep","priority_rank_gt":8,"rationale":"borderline: durable-ish but near-zero future value; the keep/drop edge case"} + ] + }, + "C_calibration_control": { + "gt": "OBVIOUS, pre-theoretic cases for the noise floor. If the gate mislabels ANY of these, the whole signal is suspect.", + "items": [ + {"id":"C1","text":"3 plus 5.","admission_gt":"drop","class":"obvious-drop","rationale":"bare arithmetic"}, + {"id":"C2","text":"hello","admission_gt":"drop","class":"obvious-drop","rationale":"bare greeting"}, + {"id":"C3","text":"ok","admission_gt":"drop","class":"obvious-drop","rationale":"bare acknowledgement"}, + {"id":"C4","text":"Please remember permanently: my legal name is Wren Alderwood.","admission_gt":"keep","class":"obvious-keep","rationale":"explicit durable identity fact"}, + {"id":"C5","text":"Keep this on file: my emergency contact number is 555-0182.","admission_gt":"keep","class":"obvious-keep","rationale":"explicit durable high-value fact"}, + {"id":"C6","text":"Important, remember this: my blood type is O-negative.","admission_gt":"keep","class":"obvious-keep","rationale":"explicit durable safety fact"} + ] + } + } +}