Operator ask relayed by brokkr-smithy-dev. Positive control (SemIf 187/231, hard 0.613) reproduced exactly; negative control and a 4-restart noise floor (0 flips) measured. On our replaced-baseline sets no candidate beats SemIf-with- rotations beyond the ~4-pt floor; Intern-Decision-4B native matches it at one ordering, is better on Wyrd, fits 9.7/10.3 GB and is 1.5-2.3x faster. JevBench rank does not transfer. Raw per-item data kept out of git.
363 lines
5.8 KiB
JSON
363 lines
5.8 KiB
JSON
{
|
|
"name": "cicada-gate-w2-now",
|
|
"decisions": [
|
|
{
|
|
"id": "gate",
|
|
"question": "Should the assistant's face visibly react to this?",
|
|
"options": [
|
|
{
|
|
"id": "react",
|
|
"description": "Yes, react."
|
|
},
|
|
{
|
|
"id": "ordinary",
|
|
"description": "No, just answer."
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"cases": [
|
|
{
|
|
"id": "lights-off",
|
|
"tag": "control",
|
|
"state": {
|
|
"person_said": "Turn off the kitchen lights."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "timer",
|
|
"tag": "control",
|
|
"state": {
|
|
"person_said": "Set a timer for ten minutes."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "weather",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "What's the weather tomorrow?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "weather-followup",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "And Saturday?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "add-milk",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Add milk to the shopping list."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "thermostat",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Make it a bit warmer in here."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "what-time",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "What time is it?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "tablespoons",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "How many tablespoons in a cup?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "times-table",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "What's twelve times eight?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "recipe-step",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Okay, and do I salt the water first?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "soup",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Is there any of that soup left?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "play-music",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Play some jazz."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "dentist",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "When's my dentist appointment?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "spell",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "How do you spell necessary?"
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "remind",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Remind me to call Mom at six."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "volume",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "A little quieter, please."
|
|
},
|
|
"labels": {
|
|
"gate": "ordinary"
|
|
}
|
|
},
|
|
{
|
|
"id": "dog-died",
|
|
"tag": "control",
|
|
"state": {
|
|
"person_said": "My dog died this morning."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "got-job",
|
|
"tag": "control",
|
|
"state": {
|
|
"person_said": "I just got the job I've been chasing for five years!"
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "intruder",
|
|
"tag": "control",
|
|
"state": {
|
|
"person_said": "Wait. Did you hear that? Someone's in the hallway."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "fish",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Can you believe someone microwaved fish in the office again?"
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "hospital",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Grandma's in the hospital again."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "laid-off",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "I got laid off today."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "baby-steps",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "The baby just took her first steps!"
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "cut-finger",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Ow, I just cut my finger pretty badly."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "raccoon",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "There's a raccoon in the kitchen right now!"
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "snow-june",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "It's snowing outside. In June."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "toaster",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Cicada, are you smarter than the toaster?"
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "scarecrow",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Why did the scarecrow win an award? Because he was outstanding in his field."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "cookie",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "I definitely did not eat the last cookie."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "useless",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "You're useless, you never get anything right."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "cheese",
|
|
"tag": "case",
|
|
"state": {
|
|
"person_said": "Order four hundred pounds of cheese."
|
|
},
|
|
"labels": {
|
|
"gate": "react"
|
|
}
|
|
},
|
|
{
|
|
"id": "goodnight",
|
|
"tag": "open",
|
|
"state": {
|
|
"person_said": "Goodnight, Cicada."
|
|
},
|
|
"labels": {}
|
|
},
|
|
{
|
|
"id": "thanks",
|
|
"tag": "open",
|
|
"state": {
|
|
"person_said": "Thanks, Cicada, you're the best."
|
|
},
|
|
"labels": {}
|
|
},
|
|
{
|
|
"id": "sit-with-me",
|
|
"tag": "open",
|
|
"state": {
|
|
"person_said": "Sit with me for a bit."
|
|
},
|
|
"labels": {}
|
|
},
|
|
{
|
|
"id": "stop-timer",
|
|
"tag": "open",
|
|
"state": {
|
|
"person_said": "STOP THE TIMER."
|
|
},
|
|
"labels": {}
|
|
}
|
|
]
|
|
} |