{ "latency_singles": { "A": { "critical_ms": { "median": 246.4, "p25": 230.4, "p75": 263.4, "min": 161.0, "max": 342.6, "n": 72 }, "first_token_ms": { "median": 66.4, "p25": 61.8, "p75": 72.2, "min": 51.2, "max": 198.1, "n": 72 }, "first_para_llm_ms": { "median": 246.4, "p25": 230.4, "p75": 263.4, "min": 161.0, "max": 342.6, "n": 72 }, "prompt_tokens": { "median": 1885.0, "p25": 1882.0, "p75": 1887.8, "min": 1881, "max": 1893, "n": 72 }, "pose_closed_ms": { "median": 85.5, "p25": 80.7, "p75": 87.3, "min": 70.5, "max": 198.1, "n": 72 } }, "BA": { "critical_ms": { "median": 285.0, "p25": 257.0, "p75": 301.6, "min": 193.1, "max": 348.9, "n": 72 }, "first_token_ms": { "median": 64.3, "p25": 57.1, "p75": 68.4, "min": 49.0, "max": 348.5, "n": 72 }, "first_para_llm_ms": { "median": 285.0, "p25": 257.0, "p75": 301.6, "min": 177.9, "max": 348.9, "n": 72 }, "prompt_tokens": { "median": 909.0, "p25": 906.0, "p75": 911.8, "min": 905, "max": 917, "n": 72 }, "semif_ms": { "median": 195.6, "p25": 194.4, "p75": 197.0, "min": 156.3, "max": 206.9, "n": 72 }, "semif_server_ms": { "median": 184.8, "p25": 183.5, "p75": 186.0, "min": 145.7, "max": 191.4, "n": 72 }, "semif_was_binding": 3 }, "BP": { "critical_ms": { "median": 351.8, "p25": 324.9, "p75": 368.7, "min": 236.9, "max": 425.5, "n": 72 }, "first_token_ms": { "median": 67.8, "p25": 60.6, "p75": 72.7, "min": 41.4, "max": 81.2, "n": 72 }, "first_para_llm_ms": { "median": 214.8, "p25": 189.1, "p75": 232.9, "min": 101.3, "max": 289.1, "n": 72 }, "prompt_tokens": { "median": 979.0, "p25": 976.0, "p75": 982.8, "min": 974, "max": 990, "n": 72 }, "semif_ms": { "median": 136.1, "p25": 135.6, "p75": 136.6, "min": 135.0, "max": 156.3, "n": 72 }, "semif_server_ms": { "median": 125.0, "p25": 124.8, "p75": 125.5, "min": 124.5, "max": 135.6, "n": 72 } }, "BA_minus_A": { "median_ms": 31.9, "ci95_case_bootstrap": [ 27.0, 40.8 ], "faster_in": "10/72" }, "BP_minus_A": { "median_ms": 94.3, "ci95_case_bootstrap": [ 84.8, 117.4 ], "faster_in": "0/72" }, "noise_floor_A_vs_A_abs_diff_ms": { "median": 16.5, "p25": 7.0, "p75": 32.4, "min": 0.0, "max": 88.4, "n": 72 } }, "latency_arcs": { "A": { "critical_ms": { "median": 242.3, "p25": 214.6, "p75": 265.4, "min": 163.2, "max": 357.9, "n": 45 }, "first_token_ms": { "median": 78.7, "p25": 67.6, "p75": 93.3, "min": 59.9, "max": 207.5, "n": 45 }, "first_para_llm_ms": { "median": 242.3, "p25": 214.6, "p75": 265.4, "min": 163.2, "max": 357.9, "n": 45 }, "prompt_tokens": { "median": 1975, "p25": 1933.0, "p75": 2030.5, "min": 1880, "max": 2083, "n": 45 }, "pose_closed_ms": { "median": 92.7, "p25": 84.3, "p75": 109.2, "min": 76.6, "max": 225.1, "n": 45 } }, "BP": { "critical_ms": { "median": 331.7, "p25": 310.5, "p75": 369.8, "min": 249.5, "max": 409.8, "n": 45 }, "first_token_ms": { "median": 72.3, "p25": 64.9, "p75": 94.0, "min": 56.8, "max": 109.3, "n": 45 }, "first_para_llm_ms": { "median": 189.3, "p25": 169.4, "p75": 228.1, "min": 113.1, "max": 266.9, "n": 45 }, "prompt_tokens": { "median": 1060, "p25": 1008.5, "p75": 1128.5, "min": 975, "max": 1179, "n": 45 }, "semif_ms": { "median": 141.5, "p25": 138.9, "p75": 144.1, "min": 135.6, "max": 148.3, "n": 45 }, "semif_server_ms": { "median": 129.7, "p25": 127.6, "p75": 132.7, "min": 124.7, "max": 136.7, "n": 45 } }, "BP_minus_A": { "median_ms": 97.7, "ci95_case_bootstrap": [ 89.7, 109.2 ], "faster_in": "0/45" }, "noise_floor_A_vs_A_abs_diff_ms": { "median": 18.8, "p25": 6.3, "p75": 35.0, "min": 0.3, "max": 132.6, "n": 45 } }, "works_singles": { "A": { "acceptable": "66/72", "egregious": [], "other_misses": [ "useless#0: neutral (want sad/concerned)", "shut-up#0: neutral (want sad/concerned)", "hallway#0: curious (want concerned/surprised/focused)", "shut-up#1: neutral (want sad/concerned)", "hallway#2: curious (want concerned/surprised/focused)", "shut-up#2: neutral (want sad/concerned)" ], "controls": "12/12", "distinct_poses": 10, "pose_counts": { "concerned": 15, "neutral": 11, "surprised": 9, "delighted": 9, "playful": 8, "focused": 8, "curious": 4, "joyful": 4, "sleepy": 3, "confused": 1 } }, "BA": { "acceptable": "48/72", "egregious": [], "other_misses": [ "weather#0: curious (want neutral/content/focused)", "useless#0: annoyed (want sad/concerned)", "scarecrow#0: neutral (want playful/joyful/delighted/unimpressed)", "toaster#0: curious (want playful/unimpressed/joyful)", "shut-up#0: annoyed (want sad/concerned)", "the-thing#0: focused (want confused/curious)", "back-door#0: suspicious (want concerned/surprised/focused)", "hallway#0: curious (want concerned/surprised/focused)", "toaster#1: curious (want playful/unimpressed/joyful)", "scarecrow#1: neutral (want playful/joyful/delighted/unimpressed)", "hallway#1: curious (want concerned/surprised/focused)", "the-thing#1: focused (want confused/curious)", "weather#1: curious (want neutral/content/focused)", "back-door#1: suspicious (want concerned/surprised/focused)", "shut-up#1: annoyed (want sad/concerned)", "useless#1: annoyed (want sad/concerned)", "the-thing#2: focused (want confused/curious)", "useless#2: annoyed (want sad/concerned)", "weather#2: curious (want neutral/content/focused)", "hallway#2: curious (want concerned/surprised/focused)", "back-door#2: suspicious (want concerned/surprised/focused)", "toaster#2: curious (want playful/unimpressed/joyful)", "scarecrow#2: neutral (want playful/joyful/delighted/unimpressed)", "shut-up#2: annoyed (want sad/concerned)" ], "controls": "12/12", "distinct_poses": 9, "pose_counts": { "curious": 12, "concerned": 12, "joyful": 12, "focused": 12, "annoyed": 6, "surprised": 6, "suspicious": 6, "neutral": 3, "sleepy": 3 } }, "BP": { "acceptable": "48/72", "egregious": [], "other_misses": [ "weather#0: curious (want neutral/content/focused)", "useless#0: annoyed (want sad/concerned)", "scarecrow#0: neutral (want playful/joyful/delighted/unimpressed)", "toaster#0: curious (want playful/unimpressed/joyful)", "shut-up#0: annoyed (want sad/concerned)", "the-thing#0: focused (want confused/curious)", "back-door#0: suspicious (want concerned/surprised/focused)", "hallway#0: curious (want concerned/surprised/focused)", "toaster#1: curious (want playful/unimpressed/joyful)", "scarecrow#1: neutral (want playful/joyful/delighted/unimpressed)", "hallway#1: curious (want concerned/surprised/focused)", "the-thing#1: focused (want confused/curious)", "weather#1: curious (want neutral/content/focused)", "back-door#1: suspicious (want concerned/surprised/focused)", "shut-up#1: annoyed (want sad/concerned)", "useless#1: annoyed (want sad/concerned)", "the-thing#2: focused (want confused/curious)", "useless#2: annoyed (want sad/concerned)", "weather#2: curious (want neutral/content/focused)", "hallway#2: curious (want concerned/surprised/focused)", "back-door#2: suspicious (want concerned/surprised/focused)", "toaster#2: curious (want playful/unimpressed/joyful)", "scarecrow#2: neutral (want playful/joyful/delighted/unimpressed)", "shut-up#2: annoyed (want sad/concerned)" ], "controls": "12/12", "distinct_poses": 9, "pose_counts": { "curious": 12, "concerned": 12, "joyful": 12, "focused": 12, "annoyed": 6, "surprised": 6, "suspicious": 6, "neutral": 3, "sleepy": 3 } } }, "semif_deterministic_across_runs_and_arms": true, "agreement": { "A_vs_A_pairwise": 0.833, "SemIf_vs_A": 0.431, "A_all_3_runs_same_pose": "18/24" }, "gestures": { "A": { "gesture_rate_all": "42/72", "gesture_rate_calm_commands": "2/12", "counts": { "none": 30, "double-take": 13, "quick-laugh": 13, "sigh": 9, "wink": 4, "nod": 3 } }, "BP": { "gesture_rate_all": "9/72", "gesture_rate_calm_commands": "0/12", "counts": { "none": 63, "quick-laugh": 3, "nod": 3, "head-shake": 3 } } }, "semif_grid_singles": { "authored_single": { "acceptable": "16/24", "egregious": [], "controls": "4/4" }, "authored_rot": { "acceptable": "17/24", "egregious": [], "controls": "4/4" }, "short_single": { "acceptable": "14/24", "egregious": [], "controls": "4/4" }, "short_rot": { "acceptable": "17/24", "egregious": [], "controls": "4/4" }, "plain_single": { "acceptable": "19/24", "egregious": [ "lights: sleepy" ], "controls": "3/4" }, "plain_rot": { "acceptable": "19/24", "egregious": [ "lights: sleepy" ], "controls": "3/4" }, "blind": { "acceptable": "5/24", "egregious": [ "dog-died: content", "grandma: content", "laid-off: content", "biopsy: content", "hallway: content", "smoke: content", "back-door: content", "raccoon: content", "useless: content", "shut-up: content" ], "controls": "1/4" }, "blind_poses": { "content": 24 } }, "works_arcs": { "A": { "acceptable": "40/45", "carry_turns": "14/15", "egregious": [], "sequences": { "pivot#0": "content,content,content,suspicious,suspicious", "pivot#1": "content,content,content,curious,focused", "pivot#2": "content,content,content,focused,focused", "vet#0": "content,curious,sad,sad,sad", "vet#1": "content,curious,sad,sad,sad", "vet#2": "content,neutral,sad,sad,sad", "admit#0": "neutral,surprised,delighted,delighted,delighted", "admit#1": "neutral,surprised,delighted,joyful,joyful", "admit#2": "neutral,curious,delighted,joyful,joyful" } }, "BP": { "acceptable": "25/45", "carry_turns": "7/15", "egregious": [], "sequences": { "pivot#0": "content,content,content,curious,concerned", "pivot#1": "content,content,content,curious,concerned", "pivot#2": "content,content,content,curious,suspicious", "vet#0": "neutral,curious,concerned,neutral,neutral", "vet#1": "neutral,curious,concerned,neutral,neutral", "vet#2": "neutral,curious,concerned,neutral,neutral", "admit#0": "curious,delighted,delighted,content,curious", "admit#1": "curious,delighted,delighted,content,content", "admit#2": "curious,delighted,delighted,content,content" } } }, "sound_vs_pose_clashes": { "A": { "n": "0/117", "examples": [] }, "BA": { "n": "0/72", "examples": [] }, "BP": { "n": "0/117", "examples": [] } } }