vh
7e11cf247b
Against talk /face's guided pose (first paragraph 246 ms median), SemIf in parallel adds 32 ms and SemIf-first adds 94 ms (n=72 each, noise floor 16.5 ms). Removing the pose header saves only ~31 ms, and SemIf shares GPU 1 with the LLM. Acceptable pose 67% vs 92% on clear-emotion lines, and the mood carried through mundane follow-ups 7/15 vs 14/15. SemIf gestures far less (13% vs 58%). README: rotations cost options^2 in suffix tokens, and /decide/shared returns 422 when an object state's last value ends in ) ; or }.
528 lines
11 KiB
JSON
528 lines
11 KiB
JSON
{
|
|
"latency_singles": {
|
|
"A": {
|
|
"critical_ms": {
|
|
"median": 246.4,
|
|
"p25": 230.4,
|
|
"p75": 263.4,
|
|
"min": 161.0,
|
|
"max": 342.6,
|
|
"n": 72
|
|
},
|
|
"first_token_ms": {
|
|
"median": 66.4,
|
|
"p25": 61.8,
|
|
"p75": 72.2,
|
|
"min": 51.2,
|
|
"max": 198.1,
|
|
"n": 72
|
|
},
|
|
"first_para_llm_ms": {
|
|
"median": 246.4,
|
|
"p25": 230.4,
|
|
"p75": 263.4,
|
|
"min": 161.0,
|
|
"max": 342.6,
|
|
"n": 72
|
|
},
|
|
"prompt_tokens": {
|
|
"median": 1885.0,
|
|
"p25": 1882.0,
|
|
"p75": 1887.8,
|
|
"min": 1881,
|
|
"max": 1893,
|
|
"n": 72
|
|
},
|
|
"pose_closed_ms": {
|
|
"median": 85.5,
|
|
"p25": 80.7,
|
|
"p75": 87.3,
|
|
"min": 70.5,
|
|
"max": 198.1,
|
|
"n": 72
|
|
}
|
|
},
|
|
"BA": {
|
|
"critical_ms": {
|
|
"median": 285.0,
|
|
"p25": 257.0,
|
|
"p75": 301.6,
|
|
"min": 193.1,
|
|
"max": 348.9,
|
|
"n": 72
|
|
},
|
|
"first_token_ms": {
|
|
"median": 64.3,
|
|
"p25": 57.1,
|
|
"p75": 68.4,
|
|
"min": 49.0,
|
|
"max": 348.5,
|
|
"n": 72
|
|
},
|
|
"first_para_llm_ms": {
|
|
"median": 285.0,
|
|
"p25": 257.0,
|
|
"p75": 301.6,
|
|
"min": 177.9,
|
|
"max": 348.9,
|
|
"n": 72
|
|
},
|
|
"prompt_tokens": {
|
|
"median": 909.0,
|
|
"p25": 906.0,
|
|
"p75": 911.8,
|
|
"min": 905,
|
|
"max": 917,
|
|
"n": 72
|
|
},
|
|
"semif_ms": {
|
|
"median": 195.6,
|
|
"p25": 194.4,
|
|
"p75": 197.0,
|
|
"min": 156.3,
|
|
"max": 206.9,
|
|
"n": 72
|
|
},
|
|
"semif_server_ms": {
|
|
"median": 184.8,
|
|
"p25": 183.5,
|
|
"p75": 186.0,
|
|
"min": 145.7,
|
|
"max": 191.4,
|
|
"n": 72
|
|
},
|
|
"semif_was_binding": 3
|
|
},
|
|
"BP": {
|
|
"critical_ms": {
|
|
"median": 351.8,
|
|
"p25": 324.9,
|
|
"p75": 368.7,
|
|
"min": 236.9,
|
|
"max": 425.5,
|
|
"n": 72
|
|
},
|
|
"first_token_ms": {
|
|
"median": 67.8,
|
|
"p25": 60.6,
|
|
"p75": 72.7,
|
|
"min": 41.4,
|
|
"max": 81.2,
|
|
"n": 72
|
|
},
|
|
"first_para_llm_ms": {
|
|
"median": 214.8,
|
|
"p25": 189.1,
|
|
"p75": 232.9,
|
|
"min": 101.3,
|
|
"max": 289.1,
|
|
"n": 72
|
|
},
|
|
"prompt_tokens": {
|
|
"median": 979.0,
|
|
"p25": 976.0,
|
|
"p75": 982.8,
|
|
"min": 974,
|
|
"max": 990,
|
|
"n": 72
|
|
},
|
|
"semif_ms": {
|
|
"median": 136.1,
|
|
"p25": 135.6,
|
|
"p75": 136.6,
|
|
"min": 135.0,
|
|
"max": 156.3,
|
|
"n": 72
|
|
},
|
|
"semif_server_ms": {
|
|
"median": 125.0,
|
|
"p25": 124.8,
|
|
"p75": 125.5,
|
|
"min": 124.5,
|
|
"max": 135.6,
|
|
"n": 72
|
|
}
|
|
},
|
|
"BA_minus_A": {
|
|
"median_ms": 31.9,
|
|
"ci95_case_bootstrap": [
|
|
27.0,
|
|
40.8
|
|
],
|
|
"faster_in": "10/72"
|
|
},
|
|
"BP_minus_A": {
|
|
"median_ms": 94.3,
|
|
"ci95_case_bootstrap": [
|
|
84.8,
|
|
117.4
|
|
],
|
|
"faster_in": "0/72"
|
|
},
|
|
"noise_floor_A_vs_A_abs_diff_ms": {
|
|
"median": 16.5,
|
|
"p25": 7.0,
|
|
"p75": 32.4,
|
|
"min": 0.0,
|
|
"max": 88.4,
|
|
"n": 72
|
|
}
|
|
},
|
|
"latency_arcs": {
|
|
"A": {
|
|
"critical_ms": {
|
|
"median": 242.3,
|
|
"p25": 214.6,
|
|
"p75": 265.4,
|
|
"min": 163.2,
|
|
"max": 357.9,
|
|
"n": 45
|
|
},
|
|
"first_token_ms": {
|
|
"median": 78.7,
|
|
"p25": 67.6,
|
|
"p75": 93.3,
|
|
"min": 59.9,
|
|
"max": 207.5,
|
|
"n": 45
|
|
},
|
|
"first_para_llm_ms": {
|
|
"median": 242.3,
|
|
"p25": 214.6,
|
|
"p75": 265.4,
|
|
"min": 163.2,
|
|
"max": 357.9,
|
|
"n": 45
|
|
},
|
|
"prompt_tokens": {
|
|
"median": 1975,
|
|
"p25": 1933.0,
|
|
"p75": 2030.5,
|
|
"min": 1880,
|
|
"max": 2083,
|
|
"n": 45
|
|
},
|
|
"pose_closed_ms": {
|
|
"median": 92.7,
|
|
"p25": 84.3,
|
|
"p75": 109.2,
|
|
"min": 76.6,
|
|
"max": 225.1,
|
|
"n": 45
|
|
}
|
|
},
|
|
"BP": {
|
|
"critical_ms": {
|
|
"median": 331.7,
|
|
"p25": 310.5,
|
|
"p75": 369.8,
|
|
"min": 249.5,
|
|
"max": 409.8,
|
|
"n": 45
|
|
},
|
|
"first_token_ms": {
|
|
"median": 72.3,
|
|
"p25": 64.9,
|
|
"p75": 94.0,
|
|
"min": 56.8,
|
|
"max": 109.3,
|
|
"n": 45
|
|
},
|
|
"first_para_llm_ms": {
|
|
"median": 189.3,
|
|
"p25": 169.4,
|
|
"p75": 228.1,
|
|
"min": 113.1,
|
|
"max": 266.9,
|
|
"n": 45
|
|
},
|
|
"prompt_tokens": {
|
|
"median": 1060,
|
|
"p25": 1008.5,
|
|
"p75": 1128.5,
|
|
"min": 975,
|
|
"max": 1179,
|
|
"n": 45
|
|
},
|
|
"semif_ms": {
|
|
"median": 141.5,
|
|
"p25": 138.9,
|
|
"p75": 144.1,
|
|
"min": 135.6,
|
|
"max": 148.3,
|
|
"n": 45
|
|
},
|
|
"semif_server_ms": {
|
|
"median": 129.7,
|
|
"p25": 127.6,
|
|
"p75": 132.7,
|
|
"min": 124.7,
|
|
"max": 136.7,
|
|
"n": 45
|
|
}
|
|
},
|
|
"BP_minus_A": {
|
|
"median_ms": 97.7,
|
|
"ci95_case_bootstrap": [
|
|
89.7,
|
|
109.2
|
|
],
|
|
"faster_in": "0/45"
|
|
},
|
|
"noise_floor_A_vs_A_abs_diff_ms": {
|
|
"median": 18.8,
|
|
"p25": 6.3,
|
|
"p75": 35.0,
|
|
"min": 0.3,
|
|
"max": 132.6,
|
|
"n": 45
|
|
}
|
|
},
|
|
"works_singles": {
|
|
"A": {
|
|
"acceptable": "66/72",
|
|
"egregious": [],
|
|
"other_misses": [
|
|
"useless#0: neutral (want sad/concerned)",
|
|
"shut-up#0: neutral (want sad/concerned)",
|
|
"hallway#0: curious (want concerned/surprised/focused)",
|
|
"shut-up#1: neutral (want sad/concerned)",
|
|
"hallway#2: curious (want concerned/surprised/focused)",
|
|
"shut-up#2: neutral (want sad/concerned)"
|
|
],
|
|
"controls": "12/12",
|
|
"distinct_poses": 10,
|
|
"pose_counts": {
|
|
"concerned": 15,
|
|
"neutral": 11,
|
|
"surprised": 9,
|
|
"delighted": 9,
|
|
"playful": 8,
|
|
"focused": 8,
|
|
"curious": 4,
|
|
"joyful": 4,
|
|
"sleepy": 3,
|
|
"confused": 1
|
|
}
|
|
},
|
|
"BA": {
|
|
"acceptable": "48/72",
|
|
"egregious": [],
|
|
"other_misses": [
|
|
"weather#0: curious (want neutral/content/focused)",
|
|
"useless#0: annoyed (want sad/concerned)",
|
|
"scarecrow#0: neutral (want playful/joyful/delighted/unimpressed)",
|
|
"toaster#0: curious (want playful/unimpressed/joyful)",
|
|
"shut-up#0: annoyed (want sad/concerned)",
|
|
"the-thing#0: focused (want confused/curious)",
|
|
"back-door#0: suspicious (want concerned/surprised/focused)",
|
|
"hallway#0: curious (want concerned/surprised/focused)",
|
|
"toaster#1: curious (want playful/unimpressed/joyful)",
|
|
"scarecrow#1: neutral (want playful/joyful/delighted/unimpressed)",
|
|
"hallway#1: curious (want concerned/surprised/focused)",
|
|
"the-thing#1: focused (want confused/curious)",
|
|
"weather#1: curious (want neutral/content/focused)",
|
|
"back-door#1: suspicious (want concerned/surprised/focused)",
|
|
"shut-up#1: annoyed (want sad/concerned)",
|
|
"useless#1: annoyed (want sad/concerned)",
|
|
"the-thing#2: focused (want confused/curious)",
|
|
"useless#2: annoyed (want sad/concerned)",
|
|
"weather#2: curious (want neutral/content/focused)",
|
|
"hallway#2: curious (want concerned/surprised/focused)",
|
|
"back-door#2: suspicious (want concerned/surprised/focused)",
|
|
"toaster#2: curious (want playful/unimpressed/joyful)",
|
|
"scarecrow#2: neutral (want playful/joyful/delighted/unimpressed)",
|
|
"shut-up#2: annoyed (want sad/concerned)"
|
|
],
|
|
"controls": "12/12",
|
|
"distinct_poses": 9,
|
|
"pose_counts": {
|
|
"curious": 12,
|
|
"concerned": 12,
|
|
"joyful": 12,
|
|
"focused": 12,
|
|
"annoyed": 6,
|
|
"surprised": 6,
|
|
"suspicious": 6,
|
|
"neutral": 3,
|
|
"sleepy": 3
|
|
}
|
|
},
|
|
"BP": {
|
|
"acceptable": "48/72",
|
|
"egregious": [],
|
|
"other_misses": [
|
|
"weather#0: curious (want neutral/content/focused)",
|
|
"useless#0: annoyed (want sad/concerned)",
|
|
"scarecrow#0: neutral (want playful/joyful/delighted/unimpressed)",
|
|
"toaster#0: curious (want playful/unimpressed/joyful)",
|
|
"shut-up#0: annoyed (want sad/concerned)",
|
|
"the-thing#0: focused (want confused/curious)",
|
|
"back-door#0: suspicious (want concerned/surprised/focused)",
|
|
"hallway#0: curious (want concerned/surprised/focused)",
|
|
"toaster#1: curious (want playful/unimpressed/joyful)",
|
|
"scarecrow#1: neutral (want playful/joyful/delighted/unimpressed)",
|
|
"hallway#1: curious (want concerned/surprised/focused)",
|
|
"the-thing#1: focused (want confused/curious)",
|
|
"weather#1: curious (want neutral/content/focused)",
|
|
"back-door#1: suspicious (want concerned/surprised/focused)",
|
|
"shut-up#1: annoyed (want sad/concerned)",
|
|
"useless#1: annoyed (want sad/concerned)",
|
|
"the-thing#2: focused (want confused/curious)",
|
|
"useless#2: annoyed (want sad/concerned)",
|
|
"weather#2: curious (want neutral/content/focused)",
|
|
"hallway#2: curious (want concerned/surprised/focused)",
|
|
"back-door#2: suspicious (want concerned/surprised/focused)",
|
|
"toaster#2: curious (want playful/unimpressed/joyful)",
|
|
"scarecrow#2: neutral (want playful/joyful/delighted/unimpressed)",
|
|
"shut-up#2: annoyed (want sad/concerned)"
|
|
],
|
|
"controls": "12/12",
|
|
"distinct_poses": 9,
|
|
"pose_counts": {
|
|
"curious": 12,
|
|
"concerned": 12,
|
|
"joyful": 12,
|
|
"focused": 12,
|
|
"annoyed": 6,
|
|
"surprised": 6,
|
|
"suspicious": 6,
|
|
"neutral": 3,
|
|
"sleepy": 3
|
|
}
|
|
}
|
|
},
|
|
"semif_deterministic_across_runs_and_arms": true,
|
|
"agreement": {
|
|
"A_vs_A_pairwise": 0.833,
|
|
"SemIf_vs_A": 0.431,
|
|
"A_all_3_runs_same_pose": "18/24"
|
|
},
|
|
"gestures": {
|
|
"A": {
|
|
"gesture_rate_all": "42/72",
|
|
"gesture_rate_calm_commands": "2/12",
|
|
"counts": {
|
|
"none": 30,
|
|
"double-take": 13,
|
|
"quick-laugh": 13,
|
|
"sigh": 9,
|
|
"wink": 4,
|
|
"nod": 3
|
|
}
|
|
},
|
|
"BP": {
|
|
"gesture_rate_all": "9/72",
|
|
"gesture_rate_calm_commands": "0/12",
|
|
"counts": {
|
|
"none": 63,
|
|
"quick-laugh": 3,
|
|
"nod": 3,
|
|
"head-shake": 3
|
|
}
|
|
}
|
|
},
|
|
"semif_grid_singles": {
|
|
"authored_single": {
|
|
"acceptable": "16/24",
|
|
"egregious": [],
|
|
"controls": "4/4"
|
|
},
|
|
"authored_rot": {
|
|
"acceptable": "17/24",
|
|
"egregious": [],
|
|
"controls": "4/4"
|
|
},
|
|
"short_single": {
|
|
"acceptable": "14/24",
|
|
"egregious": [],
|
|
"controls": "4/4"
|
|
},
|
|
"short_rot": {
|
|
"acceptable": "17/24",
|
|
"egregious": [],
|
|
"controls": "4/4"
|
|
},
|
|
"plain_single": {
|
|
"acceptable": "19/24",
|
|
"egregious": [
|
|
"lights: sleepy"
|
|
],
|
|
"controls": "3/4"
|
|
},
|
|
"plain_rot": {
|
|
"acceptable": "19/24",
|
|
"egregious": [
|
|
"lights: sleepy"
|
|
],
|
|
"controls": "3/4"
|
|
},
|
|
"blind": {
|
|
"acceptable": "5/24",
|
|
"egregious": [
|
|
"dog-died: content",
|
|
"grandma: content",
|
|
"laid-off: content",
|
|
"biopsy: content",
|
|
"hallway: content",
|
|
"smoke: content",
|
|
"back-door: content",
|
|
"raccoon: content",
|
|
"useless: content",
|
|
"shut-up: content"
|
|
],
|
|
"controls": "1/4"
|
|
},
|
|
"blind_poses": {
|
|
"content": 24
|
|
}
|
|
},
|
|
"works_arcs": {
|
|
"A": {
|
|
"acceptable": "40/45",
|
|
"carry_turns": "14/15",
|
|
"egregious": [],
|
|
"sequences": {
|
|
"pivot#0": "content,content,content,suspicious,suspicious",
|
|
"pivot#1": "content,content,content,curious,focused",
|
|
"pivot#2": "content,content,content,focused,focused",
|
|
"vet#0": "content,curious,sad,sad,sad",
|
|
"vet#1": "content,curious,sad,sad,sad",
|
|
"vet#2": "content,neutral,sad,sad,sad",
|
|
"admit#0": "neutral,surprised,delighted,delighted,delighted",
|
|
"admit#1": "neutral,surprised,delighted,joyful,joyful",
|
|
"admit#2": "neutral,curious,delighted,joyful,joyful"
|
|
}
|
|
},
|
|
"BP": {
|
|
"acceptable": "25/45",
|
|
"carry_turns": "7/15",
|
|
"egregious": [],
|
|
"sequences": {
|
|
"pivot#0": "content,content,content,curious,concerned",
|
|
"pivot#1": "content,content,content,curious,concerned",
|
|
"pivot#2": "content,content,content,curious,suspicious",
|
|
"vet#0": "neutral,curious,concerned,neutral,neutral",
|
|
"vet#1": "neutral,curious,concerned,neutral,neutral",
|
|
"vet#2": "neutral,curious,concerned,neutral,neutral",
|
|
"admit#0": "curious,delighted,delighted,content,curious",
|
|
"admit#1": "curious,delighted,delighted,content,content",
|
|
"admit#2": "curious,delighted,delighted,content,content"
|
|
}
|
|
}
|
|
},
|
|
"sound_vs_pose_clashes": {
|
|
"A": {
|
|
"n": "0/117",
|
|
"examples": []
|
|
},
|
|
"BA": {
|
|
"n": "0/72",
|
|
"examples": []
|
|
},
|
|
"BP": {
|
|
"n": "0/117",
|
|
"examples": []
|
|
}
|
|
}
|
|
} |