vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
bb806e3596edda75f1891169816545b35d61c0a4
esh-pfi-infrastructure/services/semif-serve/bench-jev-2026-09-30/summary.json
T
vh 475d6d6bcb docs: Jev candidate bench vs SemIf (fv-ml1 GPU 3) — Intern-Decision-4B is the replacement if SemIf is displaced 
Operator ask relayed by brokkr-smithy-dev. Positive control (SemIf 187/231,
hard 0.613) reproduced exactly; negative control and a 4-restart noise floor
(0 flips) measured. On our replaced-baseline sets no candidate beats SemIf-with-
rotations beyond the ~4-pt floor; Intern-Decision-4B native matches it at one
ordering, is better on Wyrd, fits 9.7/10.3 GB and is 1.5-2.3x faster. JevBench
rank does not transfer. Raw per-item data kept out of git.
2026-09-30 05:00:45 -07:00

15323 lines
238 KiB
JSON
Raw Blame History

{
"imajev-4b-native": {
"repeats": [
"r1",
"r2",
"r3",
"r4"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"easy_n": {
"median": 48.0,
"min": 48,
"max": 48,
"n": 4
},
"original": {
"median": 0.9861111111111112,
"min": 0.9861111111111112,
"max": 0.9861111111111112,
"n": 4
},
"original_n": {
"median": 71.0,
"min": 71,
"max": 71,
"n": 4
},
"hard": {
"median": 0.7207207207207207,
"min": 0.7207207207207207,
"max": 0.7207207207207207,
"n": 4
},
"hard_n": {
"median": 80.0,
"min": 80,
"max": 80,
"n": 4
},
"all": {
"median": 0.8614718614718615,
"min": 0.8614718614718615,
"max": 0.8614718614718615,
"n": 4
},
"all_n": {
"median": 199.0,
"min": 199,
"max": 199,
"n": 4
},
"failed": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
},
"p50_ms": {
"median": 62.33004853129387,
"min": 61.600604094564915,
"max": 63.11454996466637,
"n": 4
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 235.0,
"min": 235,
"max": 235,
"n": 4
},
"n": 259,
"acc": {
"median": 0.9073359073359073,
"min": 0.9073359073359073,
"max": 0.9073359073359073,
"n": 4
},
"blind_correct": {
"median": 64.0,
"min": 64,
"max": 64,
"n": 4
},
"controls": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"controls_n": 33,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/authored144": {
"correct": {
"median": 138.0,
"min": 138,
"max": 138,
"n": 4
},
"n": 144,
"acc": {
"median": 0.9583333333333334,
"min": 0.9583333333333334,
"max": 0.9583333333333334,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/perturbations108": {
"correct": {
"median": 102.0,
"min": 102,
"max": 102,
"n": 4
},
"n": 108,
"acc": {
"median": 0.9444444444444444,
"min": 0.9444444444444444,
"max": 0.9444444444444444,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/cicada-w1": {
"correct": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"n": 31,
"acc": {
"median": 0.9354838709677419,
"min": 0.9354838709677419,
"max": 0.9354838709677419,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/cicada-w2": {
"correct": {
"median": 22.0,
"min": 22,
"max": 22,
"n": 4
},
"n": 31,
"acc": {
"median": 0.7096774193548387,
"min": 0.7096774193548387,
"max": 0.7096774193548387,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd": {
"correct": {
"median": 68.0,
"min": 68,
"max": 68,
"n": 4
},
"n": 84,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 4
},
"blind_correct": {
"median": 48.0,
"min": 48,
"max": 48,
"n": 4
},
"controls": {
"median": 24.0,
"min": 24,
"max": 24,
"n": 4
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:place": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:place2": {
"correct": {
"median": 20.0,
"min": 20,
"max": 20,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 4
},
"blind_correct": {
"median": 10.0,
"min": 10,
"max": 10,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:exit": {
"correct": {
"median": 13.0,
"min": 13,
"max": 13,
"n": 4
},
"n": 21,
"acc": {
"median": 0.6190476190476191,
"min": 0.6190476190476191,
"max": 0.6190476190476191,
"n": 4
},
"blind_correct": {
"median": 8.0,
"min": 8,
"max": 8,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:exit2": {
"correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/pooled": {
"correct": {
"median": 236.0,
"min": 236,
"max": 236,
"n": 4
},
"n": 259,
"acc": {
"median": 0.9111969111969112,
"min": 0.9111969111969112,
"max": 0.9111969111969112,
"n": 4
},
"blind_correct": {
"median": 73.0,
"min": 73,
"max": 73,
"n": 4
},
"controls": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"controls_n": 33,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/authored144": {
"correct": {
"median": 138.0,
"min": 138,
"max": 138,
"n": 4
},
"n": 144,
"acc": {
"median": 0.9583333333333334,
"min": 0.9583333333333334,
"max": 0.9583333333333334,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/perturbations108": {
"correct": {
"median": 101.0,
"min": 101,
"max": 101,
"n": 4
},
"n": 108,
"acc": {
"median": 0.9351851851851852,
"min": 0.9351851851851852,
"max": 0.9351851851851852,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/cicada-w1": {
"correct": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"n": 31,
"acc": {
"median": 0.9354838709677419,
"min": 0.9354838709677419,
"max": 0.9354838709677419,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/cicada-w2": {
"correct": {
"median": 23.0,
"min": 23,
"max": 23,
"n": 4
},
"n": 31,
"acc": {
"median": 0.7419354838709677,
"min": 0.7419354838709677,
"max": 0.7419354838709677,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd": {
"correct": {
"median": 69.0,
"min": 69,
"max": 69,
"n": 4
},
"n": 84,
"acc": {
"median": 0.8214285714285714,
"min": 0.8214285714285714,
"max": 0.8214285714285714,
"n": 4
},
"blind_correct": {
"median": 57.0,
"min": 57,
"max": 57,
"n": 4
},
"controls": {
"median": 24.0,
"min": 24,
"max": 24,
"n": 4
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:place": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 20.0,
"min": 20,
"max": 20,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 4
},
"blind_correct": {
"median": 12.0,
"min": 12,
"max": 12,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 13.0,
"min": 13,
"max": 13,
"n": 4
},
"n": 21,
"acc": {
"median": 0.6190476190476191,
"min": 0.6190476190476191,
"max": 0.6190476190476191,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 17.0,
"min": 17,
"max": 17,
"n": 4
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"multifield/pooled": {
"correct": {
"median": 68,
"min": 68,
"max": 68,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 24,
"min": 24,
"max": 24,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 68,
"min": 68,
"max": 68,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 24,
"min": 24,
"max": 24,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 19,
"min": 19,
"max": 19,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 13,
"min": 13,
"max": 13,
"n": 3
},
"n": 21,
"acc": {
"median": 0.6190476190476191,
"min": 0.6190476190476191,
"max": 0.6190476190476191,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
236,
244
],
"split": [
3,
8
]
},
{
"unanimous": [
236,
244
],
"split": [
3,
8
]
},
{
"unanimous": [
236,
244
],
"split": [
3,
8
]
},
{
"unanimous": [
236,
244
],
"split": [
3,
8
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r3-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 4
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r3-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 4
}
},
"order": {
"reversed": {
"label_changes": {
"median": 3.0,
"min": 3,
"max": 3,
"n": 4
},
"n": 144,
"max_dp": {
"median": 0.7184792324116476,
"min": 0.7184792324116476,
"max": 0.7184792324116476,
"n": 4
},
"median_row_max_dp": {
"median": 0.013997185989896022,
"min": 0.013997185989896022,
"max": 0.013997185989896022,
"n": 4
},
"acc": {
"median": 139.0,
"min": 139,
"max": 139,
"n": 4
}
},
"shuffled": {
"label_changes": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"n": 144,
"max_dp": {
"median": 0.6801944591353583,
"min": 0.6801944591353583,
"max": 0.6801944591353583,
"n": 4
},
"median_row_max_dp": {
"median": 0.011850765421184902,
"min": 0.011850765421184902,
"max": 0.011850765421184902,
"n": 4
},
"acc": {
"median": 138.0,
"min": 138,
"max": 138,
"n": 4
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"vs_original_gold": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"follows_description": {
"median": 135.0,
"min": 135,
"max": 135,
"n": 4
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 10404.0,
"min": 10404.0,
"max": 10404.0,
"n": 4
},
"peak_work_mib": {
"median": 11056.0,
"min": 11056.0,
"max": 11056.0,
"n": 4
},
"peak_capacity_mib": {
"median": 11056.0,
"min": 11056.0,
"max": 11056.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 11056.0,
"min": 11056.0,
"max": 11056.0,
"n": 4
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 60.4,
"min": 60.0,
"max": 60.9,
"n": 3
},
"server_ms": {
"median": 58.4,
"min": 58.1,
"max": 58.8,
"n": 3
},
"run_medians_ms": [
59.4,
61.0
],
"tokens": 124,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 1219.1,
"min": 1211.6,
"max": 1229.8,
"n": 3
},
"server_ms": {
"median": 1212.0,
"min": 1204.3,
"max": 1222.6,
"n": 3
},
"run_medians_ms": [
1209.8,
1233.9
],
"tokens": 374,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 314.9,
"min": 310.2,
"max": 315.4,
"n": 3
},
"server_ms": {
"median": 312.6,
"min": 308.0,
"max": 313.2,
"n": 3
},
"run_medians_ms": [
309.6,
316.1
],
"tokens": 3962,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 5125.3,
"min": 5053.6,
"max": 5139.9,
"n": 3
},
"server_ms": {
"median": 5120.6,
"min": 5048.6,
"max": 5135.0,
"n": 3
},
"run_medians_ms": [
5028.7,
5142.9
],
"tokens": 7925,
"non_200": 0
}
},
"capacity": {
"r4": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
200
],
[
128,
200
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
200
],
[
20,
200
],
[
24,
200
],
[
32,
200
],
[
48,
200
],
[
64,
200
]
]
}
},
"parity_vs_upstream_semif": [],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 235,
"fixed": 30,
"broken": 12,
"mcnemar_p": 0.0079,
"delta_pts": 6.9,
"delta_95ci_pts": [
1.6,
12.2
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 138,
"fixed": 25,
"broken": 3,
"mcnemar_p": 0.0,
"delta_pts": 15.3,
"delta_95ci_pts": [
9.0,
21.5
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 102,
"fixed": 20,
"broken": 1,
"mcnemar_p": 0.0,
"delta_pts": 17.6,
"delta_95ci_pts": [
8.3,
27.8
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 22,
"fixed": 3,
"broken": 1,
"mcnemar_p": 0.625,
"delta_pts": 6.5,
"delta_95ci_pts": [
-6.5,
19.4
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 68,
"fixed": 5,
"broken": 8,
"mcnemar_p": 0.5811,
"delta_pts": -3.6,
"delta_95ci_pts": [
-13.1,
6.0
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 20,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -4.8,
"delta_95ci_pts": [
-14.3,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 13,
"fixed": 1,
"broken": 4,
"mcnemar_p": 0.375,
"delta_pts": -14.3,
"delta_95ci_pts": [
-33.3,
4.8
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 236,
"fixed": 14,
"broken": 10,
"mcnemar_p": 0.5413,
"delta_pts": 1.5,
"delta_95ci_pts": [
-2.5,
5.6
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 138,
"fixed": 8,
"broken": 1,
"mcnemar_p": 0.0391,
"delta_pts": 4.9,
"delta_95ci_pts": [
0.7,
9.0
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 101,
"fixed": 9,
"broken": 1,
"mcnemar_p": 0.0215,
"delta_pts": 7.4,
"delta_95ci_pts": [
1.9,
13.9
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 23,
"fixed": 4,
"broken": 0,
"mcnemar_p": 0.125,
"delta_pts": 12.9,
"delta_95ci_pts": [
3.2,
25.8
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 69,
"fixed": 6,
"broken": 8,
"mcnemar_p": 0.7905,
"delta_pts": -2.4,
"delta_95ci_pts": [
-11.9,
7.1
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 20,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -4.8,
"delta_95ci_pts": [
-14.3,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 13,
"fixed": 0,
"broken": 5,
"mcnemar_p": 0.0625,
"delta_pts": -23.8,
"delta_95ci_pts": [
-42.9,
-9.5
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 235,
"fixed": 15,
"broken": 12,
"mcnemar_p": 0.7011,
"delta_pts": 1.2,
"delta_95ci_pts": [
-3.2,
5.5
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 138,
"fixed": 9,
"broken": 2,
"mcnemar_p": 0.0654,
"delta_pts": 4.9,
"delta_95ci_pts": [
0.7,
9.7
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 102,
"fixed": 10,
"broken": 1,
"mcnemar_p": 0.0117,
"delta_pts": 8.3,
"delta_95ci_pts": [
1.9,
15.7
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 22,
"fixed": 3,
"broken": 0,
"mcnemar_p": 0.25,
"delta_pts": 9.7,
"delta_95ci_pts": [
0.0,
22.6
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 68,
"fixed": 6,
"broken": 9,
"mcnemar_p": 0.6072,
"delta_pts": -3.6,
"delta_95ci_pts": [
-14.3,
6.0
]
}
}
},
"intern-decision-4b-dropin": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9583333333333334,
"min": 0.9583333333333334,
"max": 0.9583333333333334,
"n": 3
},
"original_n": {
"median": 69,
"min": 69,
"max": 69,
"n": 3
},
"hard": {
"median": 0.7657657657657657,
"min": 0.7657657657657657,
"max": 0.7657657657657657,
"n": 3
},
"hard_n": {
"median": 85,
"min": 85,
"max": 85,
"n": 3
},
"all": {
"median": 0.8744588744588745,
"min": 0.8744588744588745,
"max": 0.8744588744588745,
"n": 3
},
"all_n": {
"median": 202,
"min": 202,
"max": 202,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 35.99118813872337,
"min": 35.73837806470692,
"max": 36.153115099295974,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 232,
"min": 232,
"max": 232,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8957528957528957,
"min": 0.8957528957528957,
"max": 0.8957528957528957,
"n": 3
},
"blind_correct": {
"median": 76,
"min": 76,
"max": 76,
"n": 3
},
"controls": {
"median": 28,
"min": 28,
"max": 28,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 128,
"min": 128,
"max": 128,
"n": 3
},
"n": 144,
"acc": {
"median": 0.8888888888888888,
"min": 0.8888888888888888,
"max": 0.8888888888888888,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 102,
"min": 102,
"max": 102,
"n": 3
},
"n": 108,
"acc": {
"median": 0.9444444444444444,
"min": 0.9444444444444444,
"max": 0.9444444444444444,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 19,
"min": 19,
"max": 19,
"n": 3
},
"n": 31,
"acc": {
"median": 0.6129032258064516,
"min": 0.6129032258064516,
"max": 0.6129032258064516,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 74,
"min": 74,
"max": 74,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8809523809523809,
"min": 0.8809523809523809,
"max": 0.8809523809523809,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 23,
"min": 23,
"max": 23,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 233,
"min": 233,
"max": 233,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8996138996138996,
"min": 0.8996138996138996,
"max": 0.8996138996138996,
"n": 3
},
"blind_correct": {
"median": 76,
"min": 76,
"max": 76,
"n": 3
},
"controls": {
"median": 27,
"min": 27,
"max": 27,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 130,
"min": 130,
"max": 130,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9027777777777778,
"min": 0.9027777777777778,
"max": 0.9027777777777778,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 102,
"min": 102,
"max": 102,
"n": 3
},
"n": 108,
"acc": {
"median": 0.9444444444444444,
"min": 0.9444444444444444,
"max": 0.9444444444444444,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 19,
"min": 19,
"max": 19,
"n": 3
},
"n": 31,
"acc": {
"median": 0.6129032258064516,
"min": 0.6129032258064516,
"max": 0.6129032258064516,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 73,
"min": 73,
"max": 73,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8690476190476191,
"min": 0.8690476190476191,
"max": 0.8690476190476191,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 22,
"min": 22,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 74,
"min": 74,
"max": 74,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8809523809523809,
"min": 0.8809523809523809,
"max": 0.8809523809523809,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 23,
"min": 23,
"max": 23,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 74,
"min": 74,
"max": 74,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8809523809523809,
"min": 0.8809523809523809,
"max": 0.8809523809523809,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 23,
"min": 23,
"max": 23,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
216,
227
],
"split": [
16,
25
]
},
{
"unanimous": [
217,
228
],
"split": [
15,
24
]
},
{
"unanimous": [
217,
228
],
"split": [
15,
24
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.1282364637356398
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.1282364637356398
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.03907469871182845
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.03907469871182845
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 10,
"min": 10,
"max": 10,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.9844398639960499,
"min": 0.9808733564365462,
"max": 0.9844398639960499,
"n": 3
},
"median_row_max_dp": {
"median": 0.006593655590315196,
"min": 0.006593655590315196,
"max": 0.007669849060158125,
"n": 3
},
"acc": {
"median": 126,
"min": 126,
"max": 126,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 12,
"min": 11,
"max": 12,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.9385439903393278,
"min": 0.9310779215106242,
"max": 0.9385439903393278,
"n": 3
},
"median_row_max_dp": {
"median": 0.005362038529314213,
"min": 0.004880756306387489,
"max": 0.005362038529314213,
"n": 3
},
"acc": {
"median": 128,
"min": 127,
"max": 128,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"vs_original_gold": {
"median": 10,
"min": 10,
"max": 10,
"n": 3
},
"follows_description": {
"median": 127,
"min": 127,
"max": 127,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
},
"peak_work_mib": {
"median": 12918.0,
"min": 12918.0,
"max": 12918.0,
"n": 3
},
"peak_capacity_mib": {
"median": 12920.0,
"min": 12920.0,
"max": 12920.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 37.6,
"min": 37.3,
"max": 38.2,
"n": 3
},
"server_ms": {
"median": 34.7,
"min": 34.5,
"max": 35.3,
"n": 3
},
"run_medians_ms": [
36.9,
38.3
],
"tokens": 114,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 131.8,
"min": 131.6,
"max": 132.2,
"n": 3
},
"server_ms": {
"median": 120.0,
"min": 119.8,
"max": 120.2,
"n": 3
},
"run_medians_ms": [
131.3,
133.2
],
"tokens": 62,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 186.1,
"min": 186.0,
"max": 188.0,
"n": 3
},
"server_ms": {
"median": 182.6,
"min": 182.1,
"max": 184.2,
"n": 3
},
"run_medians_ms": [
185.3,
188.4
],
"tokens": 3952,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 507.9,
"min": 500.4,
"max": 516.4,
"n": 3
},
"server_ms": {
"median": 488.5,
"min": 481.0,
"max": 496.7,
"n": 3
},
"run_medians_ms": [
499.6,
517.8
],
"tokens": 3900,
"non_200": 0
}
},
"capacity": {
"r1": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
422
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
503
]
]
}
},
"parity_vs_upstream_semif": [
{
"n": 144,
"top_agree": 124,
"prompt_sha_equal": 144,
"max_dp": 0.8729834383144334
},
{
"n": 144,
"top_agree": 124,
"prompt_sha_equal": 144,
"max_dp": 0.8730472514479778
},
{
"n": 144,
"top_agree": 124,
"prompt_sha_equal": 144,
"max_dp": 0.8730472514479778
}
],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 232,
"fixed": 18,
"broken": 3,
"mcnemar_p": 0.0015,
"delta_pts": 5.8,
"delta_95ci_pts": [
2.7,
9.0
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 128,
"fixed": 15,
"broken": 3,
"mcnemar_p": 0.0075,
"delta_pts": 8.3,
"delta_95ci_pts": [
3.5,
13.9
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 102,
"fixed": 20,
"broken": 1,
"mcnemar_p": 0.0,
"delta_pts": 17.6,
"delta_95ci_pts": [
7.4,
28.7
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 19,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 74,
"fixed": 3,
"broken": 0,
"mcnemar_p": 0.25,
"delta_pts": 3.6,
"delta_95ci_pts": [
0.0,
8.3
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 17,
"fixed": 1,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 4.8,
"delta_95ci_pts": [
0.0,
14.3
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 233,
"fixed": 8,
"broken": 7,
"mcnemar_p": 1.0,
"delta_pts": 0.4,
"delta_95ci_pts": [
-2.3,
3.1
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 130,
"fixed": 5,
"broken": 6,
"mcnemar_p": 1.0,
"delta_pts": -0.7,
"delta_95ci_pts": [
-4.9,
3.5
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 102,
"fixed": 9,
"broken": 0,
"mcnemar_p": 0.0039,
"delta_pts": 8.3,
"delta_95ci_pts": [
2.8,
15.7
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 19,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 73,
"fixed": 3,
"broken": 1,
"mcnemar_p": 0.625,
"delta_pts": 2.4,
"delta_95ci_pts": [
-2.4,
7.1
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 17,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -4.8,
"delta_95ci_pts": [
-14.3,
0.0
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 232,
"fixed": 10,
"broken": 10,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
-3.0,
3.1
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 128,
"fixed": 6,
"broken": 9,
"mcnemar_p": 0.6072,
"delta_pts": -2.1,
"delta_95ci_pts": [
-6.2,
2.1
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 102,
"fixed": 11,
"broken": 2,
"mcnemar_p": 0.0225,
"delta_pts": 8.3,
"delta_95ci_pts": [
0.0,
18.5
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 19,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 74,
"fixed": 4,
"broken": 1,
"mcnemar_p": 0.375,
"delta_pts": 3.6,
"delta_95ci_pts": [
-1.2,
8.3
]
}
}
},
"intern-decision-4b-native": {
"repeats": [
"r1",
"r2",
"r3",
"r4"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"easy_n": {
"median": 48.0,
"min": 48,
"max": 48,
"n": 4
},
"original": {
"median": 0.9861111111111112,
"min": 0.9861111111111112,
"max": 0.9861111111111112,
"n": 4
},
"original_n": {
"median": 71.0,
"min": 71,
"max": 71,
"n": 4
},
"hard": {
"median": 0.7477477477477478,
"min": 0.7477477477477478,
"max": 0.7477477477477478,
"n": 4
},
"hard_n": {
"median": 83.0,
"min": 83,
"max": 83,
"n": 4
},
"all": {
"median": 0.8744588744588745,
"min": 0.8744588744588745,
"max": 0.8744588744588745,
"n": 4
},
"all_n": {
"median": 202.0,
"min": 202,
"max": 202,
"n": 4
},
"failed": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
},
"p50_ms": {
"median": 40.11434142012149,
"min": 39.801658131182194,
"max": 40.179003961384296,
"n": 4
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 240.0,
"min": 240,
"max": 240,
"n": 4
},
"n": 259,
"acc": {
"median": 0.9266409266409267,
"min": 0.9266409266409267,
"max": 0.9266409266409267,
"n": 4
},
"blind_correct": {
"median": 76.0,
"min": 76,
"max": 76,
"n": 4
},
"controls": {
"median": 32.0,
"min": 32,
"max": 32,
"n": 4
},
"controls_n": 33,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/authored144": {
"correct": {
"median": 132.0,
"min": 132,
"max": 132,
"n": 4
},
"n": 144,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/perturbations108": {
"correct": {
"median": 100.0,
"min": 100,
"max": 100,
"n": 4
},
"n": 108,
"acc": {
"median": 0.9259259259259259,
"min": 0.9259259259259259,
"max": 0.9259259259259259,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/cicada-w1": {
"correct": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"n": 31,
"acc": {
"median": 0.9354838709677419,
"min": 0.9354838709677419,
"max": 0.9354838709677419,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/cicada-w2": {
"correct": {
"median": 22.0,
"min": 22,
"max": 22,
"n": 4
},
"n": 31,
"acc": {
"median": 0.7096774193548387,
"min": 0.7096774193548387,
"max": 0.7096774193548387,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd": {
"correct": {
"median": 79.0,
"min": 79,
"max": 79,
"n": 4
},
"n": 84,
"acc": {
"median": 0.9404761904761905,
"min": 0.9404761904761905,
"max": 0.9404761904761905,
"n": 4
},
"blind_correct": {
"median": 60.0,
"min": 60,
"max": 60,
"n": 4
},
"controls": {
"median": 27.0,
"min": 27,
"max": 27,
"n": 4
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:place": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:place2": {
"correct": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 4
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:exit": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:exit2": {
"correct": {
"median": 20.0,
"min": 20,
"max": 20,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/pooled": {
"correct": {
"median": 236.0,
"min": 236,
"max": 236,
"n": 4
},
"n": 259,
"acc": {
"median": 0.9111969111969112,
"min": 0.9111969111969112,
"max": 0.9111969111969112,
"n": 4
},
"blind_correct": {
"median": 76.0,
"min": 76,
"max": 76,
"n": 4
},
"controls": {
"median": 30.0,
"min": 30,
"max": 30,
"n": 4
},
"controls_n": 33,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/authored144": {
"correct": {
"median": 130.0,
"min": 130,
"max": 130,
"n": 4
},
"n": 144,
"acc": {
"median": 0.9027777777777778,
"min": 0.9027777777777778,
"max": 0.9027777777777778,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/perturbations108": {
"correct": {
"median": 99.0,
"min": 99,
"max": 99,
"n": 4
},
"n": 108,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/cicada-w1": {
"correct": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"n": 31,
"acc": {
"median": 0.9354838709677419,
"min": 0.9354838709677419,
"max": 0.9354838709677419,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/cicada-w2": {
"correct": {
"median": 22.0,
"min": 22,
"max": 22,
"n": 4
},
"n": 31,
"acc": {
"median": 0.7096774193548387,
"min": 0.7096774193548387,
"max": 0.7096774193548387,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd": {
"correct": {
"median": 77.0,
"min": 77,
"max": 77,
"n": 4
},
"n": 84,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 4
},
"blind_correct": {
"median": 60.0,
"min": 60,
"max": 60,
"n": 4
},
"controls": {
"median": 25.0,
"min": 25,
"max": 25,
"n": 4
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18.0,
"min": 18,
"max": 18,
"n": 4
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 4
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 21,
"acc": {
"median": 0.9047619047619048,
"min": 0.9047619047619048,
"max": 0.9047619047619048,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"multifield/pooled": {
"correct": {
"median": 77,
"min": 77,
"max": 77,
"n": 3
},
"n": 84,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 77,
"min": 77,
"max": 77,
"n": 3
},
"n": 84,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
224,
236
],
"split": [
5,
16
]
},
{
"unanimous": [
224,
236
],
"split": [
5,
16
]
},
{
"unanimous": [
224,
236
],
"split": [
5,
16
]
},
{
"unanimous": [
224,
236
],
"split": [
5,
16
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r3-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 4
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r3-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 4
}
},
"order": {
"reversed": {
"label_changes": {
"median": 8.0,
"min": 8,
"max": 8,
"n": 4
},
"n": 144,
"max_dp": {
"median": 0.6011419370187348,
"min": 0.6011419370187348,
"max": 0.6011419370187348,
"n": 4
},
"median_row_max_dp": {
"median": 0.04312028549041608,
"min": 0.04312028549041608,
"max": 0.04312028549041608,
"n": 4
},
"acc": {
"median": 133.0,
"min": 133,
"max": 133,
"n": 4
}
},
"shuffled": {
"label_changes": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"n": 144,
"max_dp": {
"median": 0.46262699339653257,
"min": 0.46262699339653257,
"max": 0.46262699339653257,
"n": 4
},
"median_row_max_dp": {
"median": 0.038418339404812527,
"min": 0.038418339404812527,
"max": 0.038418339404812527,
"n": 4
},
"acc": {
"median": 134.0,
"min": 134,
"max": 134,
"n": 4
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 10.0,
"min": 10,
"max": 10,
"n": 4
},
"vs_original_gold": {
"median": 14.0,
"min": 14,
"max": 14,
"n": 4
},
"follows_description": {
"median": 122.0,
"min": 122,
"max": 122,
"n": 4
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 9736.0,
"min": 9736.0,
"max": 9736.0,
"n": 4
},
"peak_work_mib": {
"median": 10290.0,
"min": 10290.0,
"max": 10290.0,
"n": 4
},
"peak_capacity_mib": {
"median": 10290.0,
"min": 10290.0,
"max": 10290.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 10034.0,
"min": 10034.0,
"max": 10034.0,
"n": 4
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 39.15,
"min": 38.7,
"max": 39.4,
"n": 4
},
"server_ms": {
"median": 37.25,
"min": 36.8,
"max": 37.5,
"n": 4
},
"run_medians_ms": [
38.7,
39.7
],
"tokens": 183,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 88.2,
"min": 87.5,
"max": 88.8,
"n": 4
},
"server_ms": {
"median": 84.35,
"min": 83.7,
"max": 84.8,
"n": 4
},
"run_medians_ms": [
87.4,
89.3
],
"tokens": 1083,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 189.85,
"min": 188.7,
"max": 191.4,
"n": 4
},
"server_ms": {
"median": 187.6,
"min": 186.6,
"max": 189.1,
"n": 4
},
"run_medians_ms": [
188.1,
191.7
],
"tokens": 4021,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 215.14999999999998,
"min": 213.1,
"max": 216.5,
"n": 4
},
"server_ms": {
"median": 212.8,
"min": 210.9,
"max": 214.0,
"n": 4
},
"run_medians_ms": [
212.0,
217.1
],
"tokens": 4579,
"non_200": 0
}
},
"capacity": {
"r1": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
200
],
[
128,
200
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
200
],
[
20,
200
],
[
24,
200
],
[
32,
200
],
[
48,
200
],
[
64,
200
]
]
}
},
"parity_vs_upstream_semif": [],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 240,
"fixed": 28,
"broken": 5,
"mcnemar_p": 0.0001,
"delta_pts": 8.9,
"delta_95ci_pts": [
4.7,
13.2
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 132,
"fixed": 20,
"broken": 4,
"mcnemar_p": 0.0015,
"delta_pts": 11.1,
"delta_95ci_pts": [
4.9,
18.1
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 100,
"fixed": 20,
"broken": 3,
"mcnemar_p": 0.0005,
"delta_pts": 15.7,
"delta_95ci_pts": [
4.6,
26.9
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 22,
"fixed": 2,
"broken": 0,
"mcnemar_p": 0.5,
"delta_pts": 6.5,
"delta_95ci_pts": [
0.0,
16.1
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 79,
"fixed": 8,
"broken": 0,
"mcnemar_p": 0.0078,
"delta_pts": 9.5,
"delta_95ci_pts": [
3.6,
16.7
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 19,
"fixed": 3,
"broken": 0,
"mcnemar_p": 0.25,
"delta_pts": 14.3,
"delta_95ci_pts": [
0.0,
28.6
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 236,
"fixed": 12,
"broken": 8,
"mcnemar_p": 0.5034,
"delta_pts": 1.5,
"delta_95ci_pts": [
-2.0,
5.0
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 130,
"fixed": 5,
"broken": 6,
"mcnemar_p": 1.0,
"delta_pts": -0.7,
"delta_95ci_pts": [
-5.6,
4.2
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 99,
"fixed": 9,
"broken": 3,
"mcnemar_p": 0.146,
"delta_pts": 5.6,
"delta_95ci_pts": [
-2.8,
13.0
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 22,
"fixed": 4,
"broken": 1,
"mcnemar_p": 0.375,
"delta_pts": 9.7,
"delta_95ci_pts": [
-3.2,
22.6
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 77,
"fixed": 7,
"broken": 1,
"mcnemar_p": 0.0703,
"delta_pts": 7.1,
"delta_95ci_pts": [
1.2,
13.1
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 19,
"fixed": 2,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": 4.8,
"delta_95ci_pts": [
-9.5,
23.8
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 240,
"fixed": 14,
"broken": 6,
"mcnemar_p": 0.1153,
"delta_pts": 3.1,
"delta_95ci_pts": [
-0.4,
6.7
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 132,
"fixed": 5,
"broken": 4,
"mcnemar_p": 1.0,
"delta_pts": 0.7,
"delta_95ci_pts": [
-3.5,
5.6
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 100,
"fixed": 10,
"broken": 3,
"mcnemar_p": 0.0923,
"delta_pts": 6.5,
"delta_95ci_pts": [
-2.8,
14.8
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 22,
"fixed": 3,
"broken": 0,
"mcnemar_p": 0.25,
"delta_pts": 9.7,
"delta_95ci_pts": [
0.0,
22.6
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 79,
"fixed": 9,
"broken": 1,
"mcnemar_p": 0.0215,
"delta_pts": 9.5,
"delta_95ci_pts": [
2.4,
16.7
]
}
}
},
"jevk5-v02-dropin": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9722222222222222,
"min": 0.9722222222222222,
"max": 0.9722222222222222,
"n": 3
},
"original_n": {
"median": 70,
"min": 70,
"max": 70,
"n": 3
},
"hard": {
"median": 0.7387387387387387,
"min": 0.7387387387387387,
"max": 0.7387387387387387,
"n": 3
},
"hard_n": {
"median": 82,
"min": 82,
"max": 82,
"n": 3
},
"all": {
"median": 0.8658008658008658,
"min": 0.8658008658008658,
"max": 0.8658008658008658,
"n": 3
},
"all_n": {
"median": 200,
"min": 200,
"max": 200,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 36.22348397038877,
"min": 35.537909949198365,
"max": 36.67247691191733,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 232,
"min": 232,
"max": 232,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8957528957528957,
"min": 0.8957528957528957,
"max": 0.8957528957528957,
"n": 3
},
"blind_correct": {
"median": 75,
"min": 75,
"max": 75,
"n": 3
},
"controls": {
"median": 28,
"min": 28,
"max": 28,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 130,
"min": 130,
"max": 130,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9027777777777778,
"min": 0.9027777777777778,
"max": 0.9027777777777778,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 92,
"min": 92,
"max": 92,
"n": 3
},
"n": 108,
"acc": {
"median": 0.8518518518518519,
"min": 0.8518518518518519,
"max": 0.8518518518518519,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5483870967741935,
"min": 0.5483870967741935,
"max": 0.5483870967741935,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 72,
"min": 72,
"max": 72,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 23,
"min": 23,
"max": 23,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 237,
"min": 237,
"max": 237,
"n": 3
},
"n": 259,
"acc": {
"median": 0.915057915057915,
"min": 0.915057915057915,
"max": 0.915057915057915,
"n": 3
},
"blind_correct": {
"median": 75,
"min": 75,
"max": 75,
"n": 3
},
"controls": {
"median": 27,
"min": 27,
"max": 27,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 133,
"min": 133,
"max": 133,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9236111111111112,
"min": 0.9236111111111112,
"max": 0.9236111111111112,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 93,
"min": 93,
"max": 93,
"n": 3
},
"n": 108,
"acc": {
"median": 0.8611111111111112,
"min": 0.8611111111111112,
"max": 0.8611111111111112,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5161290322580645,
"min": 0.5161290322580645,
"max": 0.5161290322580645,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 2,
"min": 2,
"max": 2,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 74,
"min": 74,
"max": 74,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8809523809523809,
"min": 0.8809523809523809,
"max": 0.8809523809523809,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 22,
"min": 22,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 73,
"min": 73,
"max": 73,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8690476190476191,
"min": 0.8690476190476191,
"max": 0.8690476190476191,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 23,
"min": 23,
"max": 23,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 73,
"min": 73,
"max": 73,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8690476190476191,
"min": 0.8690476190476191,
"max": 0.8690476190476191,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 23,
"min": 23,
"max": 23,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
204,
217
],
"split": [
22,
35
]
},
{
"unanimous": [
204,
217
],
"split": [
22,
35
]
},
{
"unanimous": [
204,
217
],
"split": [
22,
35
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.7020455918914951,
"min": 0.7020455918914951,
"max": 0.7020455918914951,
"n": 3
},
"median_row_max_dp": {
"median": 0.039757695590980066,
"min": 0.039757695590980066,
"max": 0.039757695590980066,
"n": 3
},
"acc": {
"median": 130,
"min": 130,
"max": 130,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.6956954090936615,
"min": 0.6956954090936615,
"max": 0.6956954090936615,
"n": 3
},
"median_row_max_dp": {
"median": 0.03329188632426887,
"min": 0.03329188632426887,
"max": 0.03329188632426887,
"n": 3
},
"acc": {
"median": 132,
"min": 132,
"max": 132,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"vs_original_gold": {
"median": 9,
"min": 9,
"max": 9,
"n": 3
},
"follows_description": {
"median": 128,
"min": 128,
"max": 128,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
},
"peak_work_mib": {
"median": 12918.0,
"min": 12918.0,
"max": 12918.0,
"n": 3
},
"peak_capacity_mib": null,
"peak_jevbench_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 37.6,
"min": 37.6,
"max": 37.8,
"n": 3
},
"server_ms": {
"median": 34.8,
"min": 34.7,
"max": 34.8,
"n": 3
},
"run_medians_ms": [
37.2,
37.8
],
"tokens": 114,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 132.2,
"min": 131.8,
"max": 132.4,
"n": 3
},
"server_ms": {
"median": 120.3,
"min": 119.3,
"max": 120.4,
"n": 3
},
"run_medians_ms": [
131.5,
132.8
],
"tokens": 62,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 187.2,
"min": 185.4,
"max": 187.7,
"n": 3
},
"server_ms": {
"median": 183.5,
"min": 182.0,
"max": 183.9,
"n": 3
},
"run_medians_ms": [
183.7,
187.9
],
"tokens": 3952,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 518.2,
"min": 505.0,
"max": 519.0,
"n": 3
},
"server_ms": {
"median": 498.6,
"min": 485.9,
"max": 499.6,
"n": 3
},
"run_medians_ms": [
504.2,
519.9
],
"tokens": 3900,
"non_200": 0
}
},
"capacity": {},
"parity_vs_upstream_semif": [
{
"n": 144,
"top_agree": 113,
"prompt_sha_equal": 144,
"max_dp": 0.8622559306866379
},
{
"n": 144,
"top_agree": 113,
"prompt_sha_equal": 144,
"max_dp": 0.8622559306866379
},
{
"n": 144,
"top_agree": 113,
"prompt_sha_equal": 144,
"max_dp": 0.8622559306866379
}
],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 232,
"fixed": 24,
"broken": 9,
"mcnemar_p": 0.0135,
"delta_pts": 5.8,
"delta_95ci_pts": [
1.1,
10.7
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 130,
"fixed": 21,
"broken": 7,
"mcnemar_p": 0.0125,
"delta_pts": 9.7,
"delta_95ci_pts": [
2.1,
17.4
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 92,
"fixed": 13,
"broken": 4,
"mcnemar_p": 0.049,
"delta_pts": 8.3,
"delta_95ci_pts": [
-0.9,
17.6
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 17,
"fixed": 0,
"broken": 3,
"mcnemar_p": 0.25,
"delta_pts": -9.7,
"delta_95ci_pts": [
-22.6,
0.0
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 72,
"fixed": 3,
"broken": 2,
"mcnemar_p": 1.0,
"delta_pts": 1.2,
"delta_95ci_pts": [
-3.6,
6.0
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 17,
"fixed": 2,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": 4.8,
"delta_95ci_pts": [
-9.5,
19.0
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 237,
"fixed": 11,
"broken": 6,
"mcnemar_p": 0.3323,
"delta_pts": 1.9,
"delta_95ci_pts": [
-1.1,
5.1
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 133,
"fixed": 6,
"broken": 4,
"mcnemar_p": 0.7539,
"delta_pts": 1.4,
"delta_95ci_pts": [
-2.8,
5.6
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 93,
"fixed": 4,
"broken": 4,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
-7.4,
6.5
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 16,
"fixed": 0,
"broken": 3,
"mcnemar_p": 0.25,
"delta_pts": -9.7,
"delta_95ci_pts": [
-19.4,
0.0
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 74,
"fixed": 5,
"broken": 2,
"mcnemar_p": 0.4531,
"delta_pts": 3.6,
"delta_95ci_pts": [
-2.4,
10.7
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 20,
"fixed": 3,
"broken": 1,
"mcnemar_p": 0.625,
"delta_pts": 9.5,
"delta_95ci_pts": [
-9.5,
28.6
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 232,
"fixed": 10,
"broken": 10,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
-3.2,
3.2
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 130,
"fixed": 7,
"broken": 8,
"mcnemar_p": 1.0,
"delta_pts": -0.7,
"delta_95ci_pts": [
-5.6,
4.2
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 92,
"fixed": 4,
"broken": 5,
"mcnemar_p": 1.0,
"delta_pts": -0.9,
"delta_95ci_pts": [
-8.3,
6.5
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 17,
"fixed": 0,
"broken": 2,
"mcnemar_p": 0.5,
"delta_pts": -6.5,
"delta_95ci_pts": [
-16.1,
0.0
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 72,
"fixed": 3,
"broken": 2,
"mcnemar_p": 1.0,
"delta_pts": 1.2,
"delta_95ci_pts": [
-3.6,
6.0
]
}
}
},
"jevk5-v02-native": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9583333333333334,
"min": 0.9583333333333334,
"max": 0.9583333333333334,
"n": 3
},
"original_n": {
"median": 69,
"min": 69,
"max": 69,
"n": 3
},
"hard": {
"median": 0.7297297297297297,
"min": 0.7297297297297297,
"max": 0.7387387387387387,
"n": 3
},
"hard_n": {
"median": 81,
"min": 81,
"max": 82,
"n": 3
},
"all": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8614718614718615,
"n": 3
},
"all_n": {
"median": 198,
"min": 198,
"max": 199,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 20.45961911790073,
"min": 20.339778857305646,
"max": 20.55299119092524,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 229,
"min": 228,
"max": 229,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8841698841698842,
"min": 0.8803088803088803,
"max": 0.8841698841698842,
"n": 3
},
"blind_correct": {
"median": 68,
"min": 68,
"max": 68,
"n": 3
},
"controls": {
"median": 27,
"min": 26,
"max": 27,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 132,
"min": 132,
"max": 132,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 85,
"min": 85,
"max": 86,
"n": 3
},
"n": 108,
"acc": {
"median": 0.7870370370370371,
"min": 0.7870370370370371,
"max": 0.7962962962962963,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5161290322580645,
"min": 0.5161290322580645,
"max": 0.5161290322580645,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 2,
"min": 2,
"max": 2,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 67,
"min": 66,
"max": 67,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7976190476190477,
"min": 0.7857142857142857,
"max": 0.7976190476190477,
"n": 3
},
"blind_correct": {
"median": 53,
"min": 53,
"max": 53,
"n": 3
},
"controls": {
"median": 22,
"min": 21,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 12,
"min": 11,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5238095238095238,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": {
"median": 8,
"min": 8,
"max": 8,
"n": 3
},
"controls": {
"median": 5,
"min": 4,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 228,
"min": 228,
"max": 228,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8803088803088803,
"min": 0.8803088803088803,
"max": 0.8803088803088803,
"n": 3
},
"blind_correct": {
"median": 75,
"min": 75,
"max": 75,
"n": 3
},
"controls": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 131,
"min": 131,
"max": 131,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9097222222222222,
"min": 0.9097222222222222,
"max": 0.9097222222222222,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 91,
"min": 90,
"max": 91,
"n": 3
},
"n": 108,
"acc": {
"median": 0.8425925925925926,
"min": 0.8333333333333334,
"max": 0.8425925925925926,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5161290322580645,
"min": 0.5161290322580645,
"max": 0.5161290322580645,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 2,
"min": 2,
"max": 2,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 67,
"min": 67,
"max": 67,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7976190476190477,
"min": 0.7976190476190477,
"max": 0.7976190476190477,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5714285714285714,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 67,
"min": 66,
"max": 67,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7976190476190477,
"min": 0.7857142857142857,
"max": 0.7976190476190477,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 22,
"min": 21,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 67,
"min": 66,
"max": 67,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7976190476190477,
"min": 0.7857142857142857,
"max": 0.7976190476190477,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 22,
"min": 21,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 12,
"min": 11,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5238095238095238,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 4,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
204,
219
],
"split": [
18,
33
]
},
{
"unanimous": [
204,
219
],
"split": [
18,
33
]
},
{
"unanimous": [
204,
219
],
"split": [
17,
33
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 3,
"flipped_ids": [
"854da63689519c5eeffe",
"cicada-gate-w1-now/goodnight/gate",
"wyrd-scene-fletcher/f-idle-look/exit"
],
"max_dp": 0.03637552261352539
},
{
"pair": "r2-r3",
"n": 560,
"flips": 3,
"flipped_ids": [
"854da63689519c5eeffe",
"cicada-gate-w1-now/goodnight/gate",
"wyrd-scene-fletcher/f-idle-look/exit"
],
"max_dp": 0.03637552261352539
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 1,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 1,
"pooled": 1,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 2,
"flipped_ids": [
"0aa1447680e4f25862bb",
"6ed6a1f9c601b82a11d9"
],
"max_dp": 0.020319187823727047
},
{
"pair": "r2-r3",
"n": 560,
"flips": 2,
"flipped_ids": [
"0aa1447680e4f25862bb",
"6ed6a1f9c601b82a11d9"
],
"max_dp": 0.020319187823727047
}
],
"unstable_rows/rotations": {
"authored144": 1,
"perturbations108": 1,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 1,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 13,
"min": 12,
"max": 13,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.7216088101267815,
"min": 0.7216088101267815,
"max": 0.7243065610527992,
"n": 3
},
"median_row_max_dp": {
"median": 0.059985868632793427,
"min": 0.059985868632793427,
"max": 0.06184732913970947,
"n": 3
},
"acc": {
"median": 125,
"min": 125,
"max": 126,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 10,
"min": 10,
"max": 10,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.6415014015510678,
"min": 0.6269211648032069,
"max": 0.6415014015510678,
"n": 3
},
"median_row_max_dp": {
"median": 0.058128830743953586,
"min": 0.058128830743953586,
"max": 0.06386451423168182,
"n": 3
},
"acc": {
"median": 129,
"min": 129,
"max": 129,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"vs_original_gold": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"follows_description": {
"median": 114,
"min": 114,
"max": 114,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
},
"peak_work_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
},
"peak_capacity_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 16.2,
"min": 16.1,
"max": 16.2,
"n": 3
},
"server_ms": {
"median": 14.5,
"min": 14.5,
"max": 14.5,
"n": 3
},
"run_medians_ms": [
16.0,
16.4
],
"tokens": 118,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 290.9,
"min": 290.8,
"max": 292.3,
"n": 3
},
"server_ms": {
"median": 288.9,
"min": 288.9,
"max": 290.4,
"n": 3
},
"run_medians_ms": [
290.1,
292.5
],
"tokens": 2489,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 211.9,
"min": 211.7,
"max": 212.4,
"n": 3
},
"server_ms": {
"median": 210.0,
"min": 209.8,
"max": 210.5,
"n": 3
},
"run_medians_ms": [
211.6,
212.5
],
"tokens": 3956,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 3359.6,
"min": 3358.3,
"max": 3367.8,
"n": 3
},
"server_ms": {
"median": 3357.5,
"min": 3356.4,
"max": 3365.8,
"n": 3
},
"run_medians_ms": [
3350.8,
3374.3
],
"tokens": 63302,
"non_200": 0
}
},
"capacity": {
"r1": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
200
],
[
128,
200
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
200
],
[
20,
200
],
[
24,
200
],
[
32,
200
],
[
48,
200
],
[
64,
200
]
]
}
},
"parity_vs_upstream_semif": [],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 229,
"fixed": 25,
"broken": 13,
"mcnemar_p": 0.073,
"delta_pts": 4.6,
"delta_95ci_pts": [
-0.8,
9.9
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 132,
"fixed": 23,
"broken": 7,
"mcnemar_p": 0.0052,
"delta_pts": 11.1,
"delta_95ci_pts": [
2.8,
18.8
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 85,
"fixed": 10,
"broken": 8,
"mcnemar_p": 0.8145,
"delta_pts": 1.9,
"delta_95ci_pts": [
-7.4,
11.1
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 16,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 67,
"fixed": 2,
"broken": 6,
"mcnemar_p": 0.2891,
"delta_pts": -4.8,
"delta_95ci_pts": [
-10.7,
1.2
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 12,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -19.0,
"delta_95ci_pts": [
-38.1,
-4.8
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 228,
"fixed": 9,
"broken": 13,
"mcnemar_p": 0.5235,
"delta_pts": -1.5,
"delta_95ci_pts": [
-4.9,
1.9
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 131,
"fixed": 7,
"broken": 7,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
-4.9,
4.9
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 91,
"fixed": 4,
"broken": 6,
"mcnemar_p": 0.7539,
"delta_pts": -1.9,
"delta_95ci_pts": [
-10.2,
5.6
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 16,
"fixed": 0,
"broken": 3,
"mcnemar_p": 0.25,
"delta_pts": -9.7,
"delta_95ci_pts": [
-19.4,
0.0
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 67,
"fixed": 2,
"broken": 6,
"mcnemar_p": 0.2891,
"delta_pts": -4.8,
"delta_95ci_pts": [
-10.7,
1.2
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 12,
"fixed": 0,
"broken": 6,
"mcnemar_p": 0.0312,
"delta_pts": -28.6,
"delta_95ci_pts": [
-47.6,
-9.5
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 229,
"fixed": 11,
"broken": 14,
"mcnemar_p": 0.69,
"delta_pts": -1.2,
"delta_95ci_pts": [
-5.0,
2.6
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 132,
"fixed": 8,
"broken": 7,
"mcnemar_p": 1.0,
"delta_pts": 0.7,
"delta_95ci_pts": [
-4.2,
5.6
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 85,
"fixed": 4,
"broken": 12,
"mcnemar_p": 0.0768,
"delta_pts": -7.4,
"delta_95ci_pts": [
-16.7,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 16,
"fixed": 0,
"broken": 3,
"mcnemar_p": 0.25,
"delta_pts": -9.7,
"delta_95ci_pts": [
-19.4,
0.0
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 67,
"fixed": 3,
"broken": 7,
"mcnemar_p": 0.3438,
"delta_pts": -4.8,
"delta_95ci_pts": [
-13.1,
2.4
]
}
}
},
"jevk5-v03-dropin": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9444444444444444,
"min": 0.9444444444444444,
"max": 0.9444444444444444,
"n": 3
},
"original_n": {
"median": 68,
"min": 68,
"max": 68,
"n": 3
},
"hard": {
"median": 0.7657657657657657,
"min": 0.7657657657657657,
"max": 0.7657657657657657,
"n": 3
},
"hard_n": {
"median": 85,
"min": 85,
"max": 85,
"n": 3
},
"all": {
"median": 0.8701298701298701,
"min": 0.8701298701298701,
"max": 0.8701298701298701,
"n": 3
},
"all_n": {
"median": 201,
"min": 201,
"max": 201,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 36.19936900213361,
"min": 35.76278709806502,
"max": 36.32741095498204,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 227,
"min": 227,
"max": 227,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8764478764478765,
"min": 0.8764478764478765,
"max": 0.8764478764478765,
"n": 3
},
"blind_correct": {
"median": 76,
"min": 76,
"max": 76,
"n": 3
},
"controls": {
"median": 27,
"min": 27,
"max": 27,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 132,
"min": 132,
"max": 132,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 99,
"min": 99,
"max": 99,
"n": 3
},
"n": 108,
"acc": {
"median": 0.9166666666666666,
"min": 0.9166666666666666,
"max": 0.9166666666666666,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"n": 31,
"acc": {
"median": 0.8387096774193549,
"min": 0.8387096774193549,
"max": 0.8387096774193549,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 31,
"acc": {
"median": 0.6774193548387096,
"min": 0.6774193548387096,
"max": 0.6774193548387096,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 69,
"min": 69,
"max": 69,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8214285714285714,
"min": 0.8214285714285714,
"max": 0.8214285714285714,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 22,
"min": 22,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 227,
"min": 227,
"max": 227,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8764478764478765,
"min": 0.8764478764478765,
"max": 0.8764478764478765,
"n": 3
},
"blind_correct": {
"median": 76,
"min": 76,
"max": 76,
"n": 3
},
"controls": {
"median": 25,
"min": 25,
"max": 25,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 135,
"min": 135,
"max": 135,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9375,
"min": 0.9375,
"max": 0.9375,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 101,
"min": 101,
"max": 101,
"n": 3
},
"n": 108,
"acc": {
"median": 0.9351851851851852,
"min": 0.9351851851851852,
"max": 0.9351851851851852,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"n": 31,
"acc": {
"median": 0.8387096774193549,
"min": 0.8387096774193549,
"max": 0.8387096774193549,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 19,
"min": 19,
"max": 19,
"n": 3
},
"n": 31,
"acc": {
"median": 0.6129032258064516,
"min": 0.6129032258064516,
"max": 0.6129032258064516,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 13,
"min": 13,
"max": 13,
"n": 3
},
"n": 21,
"acc": {
"median": 0.6190476190476191,
"min": 0.6190476190476191,
"max": 0.6190476190476191,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 69,
"min": 69,
"max": 69,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8214285714285714,
"min": 0.8214285714285714,
"max": 0.8214285714285714,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 22,
"min": 22,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 69,
"min": 69,
"max": 69,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8214285714285714,
"min": 0.8214285714285714,
"max": 0.8214285714285714,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 22,
"min": 22,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
225,
236
],
"split": [
11,
16
]
},
{
"unanimous": [
225,
236
],
"split": [
11,
16
]
},
{
"unanimous": [
225,
236
],
"split": [
11,
16
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.5338371380556028,
"min": 0.5338371380556028,
"max": 0.5338371380556028,
"n": 3
},
"median_row_max_dp": {
"median": 0.02727540894485736,
"min": 0.02727540894485736,
"max": 0.02727540894485736,
"n": 3
},
"acc": {
"median": 132,
"min": 132,
"max": 132,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.5052658931497024,
"min": 0.5052658931497024,
"max": 0.5052658931497024,
"n": 3
},
"median_row_max_dp": {
"median": 0.02432059004338808,
"min": 0.02432059004338808,
"max": 0.02432059004338808,
"n": 3
},
"acc": {
"median": 133,
"min": 133,
"max": 133,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"vs_original_gold": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"follows_description": {
"median": 133,
"min": 133,
"max": 133,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
},
"peak_work_mib": {
"median": 12918.0,
"min": 12918.0,
"max": 12918.0,
"n": 3
},
"peak_capacity_mib": null,
"peak_jevbench_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 37.8,
"min": 37.0,
"max": 38.7,
"n": 3
},
"server_ms": {
"median": 34.8,
"min": 34.3,
"max": 35.7,
"n": 3
},
"run_medians_ms": [
36.9,
38.8
],
"tokens": 114,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 132.6,
"min": 130.8,
"max": 133.1,
"n": 3
},
"server_ms": {
"median": 120.5,
"min": 119.4,
"max": 121.0,
"n": 3
},
"run_medians_ms": [
130.4,
133.2
],
"tokens": 62,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 187.4,
"min": 185.8,
"max": 187.6,
"n": 3
},
"server_ms": {
"median": 183.5,
"min": 182.4,
"max": 183.7,
"n": 3
},
"run_medians_ms": [
183.9,
187.8
],
"tokens": 3952,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 517.7,
"min": 506.2,
"max": 520.7,
"n": 3
},
"server_ms": {
"median": 497.9,
"min": 486.9,
"max": 500.9,
"n": 3
},
"run_medians_ms": [
505.8,
522.2
],
"tokens": 3900,
"non_200": 0
}
},
"capacity": {},
"parity_vs_upstream_semif": [
{
"n": 144,
"top_agree": 114,
"prompt_sha_equal": 144,
"max_dp": 0.8999394938538358
},
{
"n": 144,
"top_agree": 114,
"prompt_sha_equal": 144,
"max_dp": 0.8999394938538358
},
{
"n": 144,
"top_agree": 114,
"prompt_sha_equal": 144,
"max_dp": 0.8999394938538358
}
],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 227,
"fixed": 24,
"broken": 14,
"mcnemar_p": 0.1433,
"delta_pts": 3.9,
"delta_95ci_pts": [
-0.8,
8.2
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 132,
"fixed": 22,
"broken": 6,
"mcnemar_p": 0.0037,
"delta_pts": 11.1,
"delta_95ci_pts": [
4.9,
17.4
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 99,
"fixed": 17,
"broken": 1,
"mcnemar_p": 0.0001,
"delta_pts": 14.8,
"delta_95ci_pts": [
6.5,
24.1
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 21,
"fixed": 1,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 3.2,
"delta_95ci_pts": [
0.0,
9.7
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 69,
"fixed": 2,
"broken": 4,
"mcnemar_p": 0.6875,
"delta_pts": -2.4,
"delta_95ci_pts": [
-7.1,
2.4
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 15,
"fixed": 1,
"broken": 2,
"mcnemar_p": 1.0,
"delta_pts": -4.8,
"delta_95ci_pts": [
-23.8,
9.5
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 227,
"fixed": 10,
"broken": 15,
"mcnemar_p": 0.4244,
"delta_pts": -1.9,
"delta_95ci_pts": [
-5.5,
1.8
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 135,
"fixed": 8,
"broken": 4,
"mcnemar_p": 0.3877,
"delta_pts": 2.8,
"delta_95ci_pts": [
-1.4,
7.6
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 101,
"fixed": 8,
"broken": 0,
"mcnemar_p": 0.0078,
"delta_pts": 7.4,
"delta_95ci_pts": [
1.9,
13.9
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 19,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 66,
"fixed": 2,
"broken": 7,
"mcnemar_p": 0.1797,
"delta_pts": -6.0,
"delta_95ci_pts": [
-11.9,
0.0
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 20,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -4.8,
"delta_95ci_pts": [
-14.3,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 13,
"fixed": 0,
"broken": 5,
"mcnemar_p": 0.0625,
"delta_pts": -23.8,
"delta_95ci_pts": [
-42.9,
-9.5
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 227,
"fixed": 10,
"broken": 15,
"mcnemar_p": 0.4244,
"delta_pts": -1.9,
"delta_95ci_pts": [
-5.2,
1.2
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 132,
"fixed": 7,
"broken": 6,
"mcnemar_p": 1.0,
"delta_pts": 0.7,
"delta_95ci_pts": [
-3.5,
4.9
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 99,
"fixed": 8,
"broken": 2,
"mcnemar_p": 0.1094,
"delta_pts": 5.6,
"delta_95ci_pts": [
0.0,
12.0
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 21,
"fixed": 2,
"broken": 0,
"mcnemar_p": 0.5,
"delta_pts": 6.5,
"delta_95ci_pts": [
0.0,
16.1
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 69,
"fixed": 3,
"broken": 5,
"mcnemar_p": 0.7266,
"delta_pts": -2.4,
"delta_95ci_pts": [
-8.3,
3.6
]
}
}
},
"jevk5-v03-native": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9444444444444444,
"min": 0.9444444444444444,
"max": 0.9444444444444444,
"n": 3
},
"original_n": {
"median": 68,
"min": 68,
"max": 68,
"n": 3
},
"hard": {
"median": 0.7747747747747747,
"min": 0.7747747747747747,
"max": 0.7747747747747747,
"n": 3
},
"hard_n": {
"median": 86,
"min": 86,
"max": 86,
"n": 3
},
"all": {
"median": 0.8744588744588745,
"min": 0.8744588744588745,
"max": 0.8744588744588745,
"n": 3
},
"all_n": {
"median": 202,
"min": 202,
"max": 202,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 20.449982024729252,
"min": 20.324888871982694,
"max": 20.55873302742839,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 227,
"min": 227,
"max": 227,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8764478764478765,
"min": 0.8764478764478765,
"max": 0.8764478764478765,
"n": 3
},
"blind_correct": {
"median": 76,
"min": 76,
"max": 76,
"n": 3
},
"controls": {
"median": 25,
"min": 25,
"max": 25,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 135,
"min": 135,
"max": 135,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9375,
"min": 0.9375,
"max": 0.9375,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 101,
"min": 101,
"max": 102,
"n": 3
},
"n": 108,
"acc": {
"median": 0.9351851851851852,
"min": 0.9351851851851852,
"max": 0.9444444444444444,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"n": 31,
"acc": {
"median": 0.8387096774193549,
"min": 0.8387096774193549,
"max": 0.8387096774193549,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5483870967741935,
"min": 0.5483870967741935,
"max": 0.5483870967741935,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5714285714285714,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 226,
"min": 226,
"max": 226,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8725868725868726,
"min": 0.8725868725868726,
"max": 0.8725868725868726,
"n": 3
},
"blind_correct": {
"median": 76,
"min": 76,
"max": 76,
"n": 3
},
"controls": {
"median": 24,
"min": 24,
"max": 24,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 135,
"min": 135,
"max": 135,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9375,
"min": 0.9375,
"max": 0.9375,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 101,
"min": 101,
"max": 101,
"n": 3
},
"n": 108,
"acc": {
"median": 0.9351851851851852,
"min": 0.9351851851851852,
"max": 0.9351851851851852,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"n": 31,
"acc": {
"median": 0.8387096774193549,
"min": 0.8387096774193549,
"max": 0.8387096774193549,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5161290322580645,
"min": 0.5161290322580645,
"max": 0.5161290322580645,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 2,
"min": 2,
"max": 2,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 65,
"min": 65,
"max": 65,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7738095238095238,
"min": 0.7738095238095238,
"max": 0.7738095238095238,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 19,
"min": 19,
"max": 19,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 21,
"acc": {
"median": 0.9523809523809523,
"min": 0.9523809523809523,
"max": 0.9523809523809523,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5714285714285714,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5714285714285714,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7142857142857143,
"min": 0.7142857142857143,
"max": 0.7142857142857143,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
227,
237
],
"split": [
9,
15
]
},
{
"unanimous": [
227,
238
],
"split": [
9,
14
]
},
{
"unanimous": [
227,
238
],
"split": [
9,
14
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 1,
"flipped_ids": [
"3cafd544b9a8b89091a4"
],
"max_dp": 0.05065256357192993
},
{
"pair": "r1-r3",
"n": 560,
"flips": 1,
"flipped_ids": [
"3cafd544b9a8b89091a4"
],
"max_dp": 0.05065256357192993
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 1,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 1,
"flipped_ids": [
"33d6e5da58f0fee2490d"
],
"max_dp": 0.0380966663066748
},
{
"pair": "r1-r3",
"n": 560,
"flips": 1,
"flipped_ids": [
"33d6e5da58f0fee2490d"
],
"max_dp": 0.0380966663066748
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 1,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 1,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 8,
"min": 7,
"max": 8,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.6086461553350091,
"min": 0.6086461553350091,
"max": 0.6087501989677548,
"n": 3
},
"median_row_max_dp": {
"median": 0.039248053915798664,
"min": 0.03914132248610258,
"max": 0.039248053915798664,
"n": 3
},
"acc": {
"median": 136,
"min": 135,
"max": 136,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 11,
"min": 11,
"max": 11,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.4905429254285991,
"min": 0.4895309181883931,
"max": 0.4905429254285991,
"n": 3
},
"median_row_max_dp": {
"median": 0.04056410677731037,
"min": 0.03994348458945751,
"max": 0.04056410677731037,
"n": 3
},
"acc": {
"median": 131,
"min": 131,
"max": 131,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"vs_original_gold": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"follows_description": {
"median": 118,
"min": 118,
"max": 118,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
},
"peak_work_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
},
"peak_capacity_mib": null,
"peak_jevbench_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 16.2,
"min": 16.1,
"max": 16.4,
"n": 3
},
"server_ms": {
"median": 14.6,
"min": 14.6,
"max": 14.8,
"n": 3
},
"run_medians_ms": [
16.1,
16.5
],
"tokens": 118,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 291.2,
"min": 291.0,
"max": 292.3,
"n": 3
},
"server_ms": {
"median": 289.3,
"min": 289.1,
"max": 290.4,
"n": 3
},
"run_medians_ms": [
290.4,
292.7
],
"tokens": 2489,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 211.8,
"min": 211.5,
"max": 212.3,
"n": 3
},
"server_ms": {
"median": 210.1,
"min": 209.7,
"max": 210.4,
"n": 3
},
"run_medians_ms": [
211.4,
212.5
],
"tokens": 3956,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 3362.8,
"min": 3354.7,
"max": 3369.6,
"n": 3
},
"server_ms": {
"median": 3360.7,
"min": 3352.6,
"max": 3367.6,
"n": 3
},
"run_medians_ms": [
3347.8,
3374.3
],
"tokens": 63302,
"non_200": 0
}
},
"capacity": {},
"parity_vs_upstream_semif": [],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 227,
"fixed": 25,
"broken": 15,
"mcnemar_p": 0.1539,
"delta_pts": 3.9,
"delta_95ci_pts": [
-0.9,
8.5
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 135,
"fixed": 24,
"broken": 5,
"mcnemar_p": 0.0005,
"delta_pts": 13.2,
"delta_95ci_pts": [
6.9,
19.4
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 101,
"fixed": 18,
"broken": 0,
"mcnemar_p": 0.0,
"delta_pts": 16.7,
"delta_95ci_pts": [
8.3,
25.9
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 17,
"fixed": 0,
"broken": 3,
"mcnemar_p": 0.25,
"delta_pts": -9.7,
"delta_95ci_pts": [
-19.4,
0.0
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 66,
"fixed": 1,
"broken": 6,
"mcnemar_p": 0.125,
"delta_pts": -6.0,
"delta_95ci_pts": [
-10.7,
-1.2
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 12,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -19.0,
"delta_95ci_pts": [
-38.1,
-4.8
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 226,
"fixed": 9,
"broken": 15,
"mcnemar_p": 0.3075,
"delta_pts": -2.3,
"delta_95ci_pts": [
-6.0,
1.4
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 135,
"fixed": 7,
"broken": 3,
"mcnemar_p": 0.3438,
"delta_pts": 2.8,
"delta_95ci_pts": [
-1.4,
7.6
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 101,
"fixed": 8,
"broken": 0,
"mcnemar_p": 0.0078,
"delta_pts": 7.4,
"delta_95ci_pts": [
1.9,
13.9
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 16,
"fixed": 0,
"broken": 3,
"mcnemar_p": 0.25,
"delta_pts": -9.7,
"delta_95ci_pts": [
-19.4,
0.0
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 65,
"fixed": 2,
"broken": 8,
"mcnemar_p": 0.1094,
"delta_pts": -7.1,
"delta_95ci_pts": [
-13.1,
-1.2
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 20,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -4.8,
"delta_95ci_pts": [
-14.3,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 12,
"fixed": 0,
"broken": 6,
"mcnemar_p": 0.0312,
"delta_pts": -28.6,
"delta_95ci_pts": [
-47.6,
-9.5
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 227,
"fixed": 11,
"broken": 16,
"mcnemar_p": 0.4421,
"delta_pts": -1.9,
"delta_95ci_pts": [
-5.5,
1.5
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 135,
"fixed": 9,
"broken": 5,
"mcnemar_p": 0.424,
"delta_pts": 2.8,
"delta_95ci_pts": [
-1.4,
6.9
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 101,
"fixed": 8,
"broken": 0,
"mcnemar_p": 0.0078,
"delta_pts": 7.4,
"delta_95ci_pts": [
1.9,
13.9
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 17,
"fixed": 0,
"broken": 2,
"mcnemar_p": 0.5,
"delta_pts": -6.5,
"delta_95ci_pts": [
-16.1,
0.0
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 66,
"fixed": 2,
"broken": 7,
"mcnemar_p": 0.1797,
"delta_pts": -6.0,
"delta_95ci_pts": [
-11.9,
0.0
]
}
}
},
"plumb-4b-dropin": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9722222222222222,
"min": 0.9722222222222222,
"max": 0.9722222222222222,
"n": 3
},
"original_n": {
"median": 70,
"min": 70,
"max": 70,
"n": 3
},
"hard": {
"median": 0.7927927927927928,
"min": 0.7927927927927928,
"max": 0.7927927927927928,
"n": 3
},
"hard_n": {
"median": 88,
"min": 88,
"max": 88,
"n": 3
},
"all": {
"median": 0.8917748917748918,
"min": 0.8917748917748918,
"max": 0.8917748917748918,
"n": 3
},
"all_n": {
"median": 206,
"min": 206,
"max": 206,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 35.559840966016054,
"min": 35.527948988601565,
"max": 35.68316902965307,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 221,
"min": 221,
"max": 221,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8532818532818532,
"min": 0.8532818532818532,
"max": 0.8532818532818532,
"n": 3
},
"blind_correct": {
"median": 75,
"min": 75,
"max": 75,
"n": 3
},
"controls": {
"median": 25,
"min": 25,
"max": 25,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 129,
"min": 129,
"max": 129,
"n": 3
},
"n": 144,
"acc": {
"median": 0.8958333333333334,
"min": 0.8958333333333334,
"max": 0.8958333333333334,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 88,
"min": 88,
"max": 88,
"n": 3
},
"n": 108,
"acc": {
"median": 0.8148148148148148,
"min": 0.8148148148148148,
"max": 0.8148148148148148,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"n": 31,
"acc": {
"median": 0.8387096774193549,
"min": 0.8387096774193549,
"max": 0.8387096774193549,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"n": 31,
"acc": {
"median": 0.6451612903225806,
"min": 0.6451612903225806,
"max": 0.6451612903225806,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5714285714285714,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 224,
"min": 224,
"max": 224,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8648648648648649,
"min": 0.8648648648648649,
"max": 0.8648648648648649,
"n": 3
},
"blind_correct": {
"median": 75,
"min": 75,
"max": 75,
"n": 3
},
"controls": {
"median": 26,
"min": 26,
"max": 26,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 129,
"min": 129,
"max": 129,
"n": 3
},
"n": 144,
"acc": {
"median": 0.8958333333333334,
"min": 0.8958333333333334,
"max": 0.8958333333333334,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 92,
"min": 92,
"max": 92,
"n": 3
},
"n": 108,
"acc": {
"median": 0.8518518518518519,
"min": 0.8518518518518519,
"max": 0.8518518518518519,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 28,
"min": 28,
"max": 28,
"n": 3
},
"n": 31,
"acc": {
"median": 0.9032258064516129,
"min": 0.9032258064516129,
"max": 0.9032258064516129,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5806451612903226,
"min": 0.5806451612903226,
"max": 0.5806451612903226,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 67,
"min": 67,
"max": 67,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7976190476190477,
"min": 0.7976190476190477,
"max": 0.7976190476190477,
"n": 3
},
"blind_correct": {
"median": 60,
"min": 60,
"max": 60,
"n": 3
},
"controls": {
"median": 22,
"min": 22,
"max": 22,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 13,
"min": 13,
"max": 13,
"n": 3
},
"n": 21,
"acc": {
"median": 0.6190476190476191,
"min": 0.6190476190476191,
"max": 0.6190476190476191,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 66,
"min": 66,
"max": 66,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7857142857142857,
"min": 0.7857142857142857,
"max": 0.7857142857142857,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5714285714285714,
"min": 0.5714285714285714,
"max": 0.5714285714285714,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
201,
221
],
"split": [
20,
31
]
},
{
"unanimous": [
201,
221
],
"split": [
20,
31
]
},
{
"unanimous": [
201,
221
],
"split": [
20,
31
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.7267885101814081,
"min": 0.7267885101814081,
"max": 0.7267885101814081,
"n": 3
},
"median_row_max_dp": {
"median": 0.017241412267327286,
"min": 0.017241412267327286,
"max": 0.017241412267327286,
"n": 3
},
"acc": {
"median": 129,
"min": 129,
"max": 129,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 8,
"min": 8,
"max": 8,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.6679675533903773,
"min": 0.6679675533903773,
"max": 0.6679675533903773,
"n": 3
},
"median_row_max_dp": {
"median": 0.014612479257242513,
"min": 0.014612479257242513,
"max": 0.014612479257242513,
"n": 3
},
"acc": {
"median": 134,
"min": 134,
"max": 134,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 3,
"min": 3,
"max": 3,
"n": 3
},
"vs_original_gold": {
"median": 12,
"min": 12,
"max": 12,
"n": 3
},
"follows_description": {
"median": 126,
"min": 126,
"max": 126,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
},
"peak_work_mib": {
"median": 12918.0,
"min": 12918.0,
"max": 12918.0,
"n": 3
},
"peak_capacity_mib": {
"median": 12920.0,
"min": 12920.0,
"max": 12920.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 37.1,
"min": 36.8,
"max": 37.3,
"n": 3
},
"server_ms": {
"median": 34.3,
"min": 34.1,
"max": 34.5,
"n": 3
},
"run_medians_ms": [
36.8,
37.5
],
"tokens": 114,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 130.9,
"min": 130.7,
"max": 131.7,
"n": 3
},
"server_ms": {
"median": 119.5,
"min": 119.1,
"max": 119.7,
"n": 3
},
"run_medians_ms": [
130.5,
132.3
],
"tokens": 62,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 186.3,
"min": 186.1,
"max": 186.4,
"n": 3
},
"server_ms": {
"median": 182.6,
"min": 182.5,
"max": 182.6,
"n": 3
},
"run_medians_ms": [
184.1,
186.8
],
"tokens": 3952,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 506.5,
"min": 506.0,
"max": 507.3,
"n": 3
},
"server_ms": {
"median": 487.4,
"min": 486.6,
"max": 488.0,
"n": 3
},
"run_medians_ms": [
505.1,
509.1
],
"tokens": 3900,
"non_200": 0
}
},
"capacity": {
"r1": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
422
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
503
]
]
}
},
"parity_vs_upstream_semif": [
{
"n": 144,
"top_agree": 116,
"prompt_sha_equal": 144,
"max_dp": 0.9170767402206121
},
{
"n": 144,
"top_agree": 116,
"prompt_sha_equal": 144,
"max_dp": 0.9170767402206121
},
{
"n": 144,
"top_agree": 116,
"prompt_sha_equal": 144,
"max_dp": 0.9170767402206121
}
],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 221,
"fixed": 21,
"broken": 17,
"mcnemar_p": 0.6271,
"delta_pts": 1.5,
"delta_95ci_pts": [
-3.7,
6.6
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 129,
"fixed": 20,
"broken": 7,
"mcnemar_p": 0.0192,
"delta_pts": 9.0,
"delta_95ci_pts": [
1.4,
16.7
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 88,
"fixed": 12,
"broken": 7,
"mcnemar_p": 0.3593,
"delta_pts": 4.6,
"delta_95ci_pts": [
-5.6,
14.8
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 20,
"fixed": 1,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
-9.7,
9.7
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 66,
"fixed": 1,
"broken": 6,
"mcnemar_p": 0.125,
"delta_pts": -6.0,
"delta_95ci_pts": [
-11.9,
0.0
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 12,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -19.0,
"delta_95ci_pts": [
-38.1,
-4.8
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 224,
"fixed": 7,
"broken": 15,
"mcnemar_p": 0.1338,
"delta_pts": -3.1,
"delta_95ci_pts": [
-6.8,
0.4
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 129,
"fixed": 5,
"broken": 7,
"mcnemar_p": 0.7744,
"delta_pts": -1.4,
"delta_95ci_pts": [
-6.2,
3.5
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 92,
"fixed": 6,
"broken": 7,
"mcnemar_p": 1.0,
"delta_pts": -0.9,
"delta_95ci_pts": [
-10.2,
7.4
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 28,
"fixed": 0,
"broken": 2,
"mcnemar_p": 0.5,
"delta_pts": -6.5,
"delta_95ci_pts": [
-16.1,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 18,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 67,
"fixed": 2,
"broken": 6,
"mcnemar_p": 0.2891,
"delta_pts": -4.8,
"delta_95ci_pts": [
-11.9,
2.4
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 13,
"fixed": 0,
"broken": 5,
"mcnemar_p": 0.0625,
"delta_pts": -23.8,
"delta_95ci_pts": [
-42.9,
-9.5
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 221,
"fixed": 8,
"broken": 19,
"mcnemar_p": 0.0522,
"delta_pts": -4.2,
"delta_95ci_pts": [
-8.3,
-0.4
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 129,
"fixed": 6,
"broken": 8,
"mcnemar_p": 0.7905,
"delta_pts": -1.4,
"delta_95ci_pts": [
-6.2,
3.5
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 88,
"fixed": 5,
"broken": 10,
"mcnemar_p": 0.3018,
"delta_pts": -4.6,
"delta_95ci_pts": [
-13.9,
3.7
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 26,
"fixed": 0,
"broken": 4,
"mcnemar_p": 0.125,
"delta_pts": -12.9,
"delta_95ci_pts": [
-25.8,
-3.2
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 20,
"fixed": 1,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 3.2,
"delta_95ci_pts": [
0.0,
9.7
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 66,
"fixed": 2,
"broken": 7,
"mcnemar_p": 0.1797,
"delta_pts": -6.0,
"delta_95ci_pts": [
-13.1,
1.2
]
}
}
},
"plumb-4b-native": {
"repeats": [
"r1",
"r2",
"r3"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"easy_n": {
"median": 48,
"min": 48,
"max": 48,
"n": 3
},
"original": {
"median": 0.9722222222222222,
"min": 0.9722222222222222,
"max": 0.9722222222222222,
"n": 3
},
"original_n": {
"median": 70,
"min": 70,
"max": 70,
"n": 3
},
"hard": {
"median": 0.8018018018018018,
"min": 0.7927927927927928,
"max": 0.8018018018018018,
"n": 3
},
"hard_n": {
"median": 89,
"min": 88,
"max": 89,
"n": 3
},
"all": {
"median": 0.8961038961038961,
"min": 0.8917748917748918,
"max": 0.8961038961038961,
"n": 3
},
"all_n": {
"median": 207,
"min": 206,
"max": 207,
"n": 3
},
"failed": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
},
"p50_ms": {
"median": 20.26381203904748,
"min": 20.13147296383977,
"max": 20.308695966377854,
"n": 3
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 224,
"min": 224,
"max": 226,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8648648648648649,
"min": 0.8648648648648649,
"max": 0.8725868725868726,
"n": 3
},
"blind_correct": {
"median": 65,
"min": 65,
"max": 65,
"n": 3
},
"controls": {
"median": 25,
"min": 25,
"max": 25,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/authored144": {
"correct": {
"median": 130,
"min": 130,
"max": 132,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9027777777777778,
"min": 0.9027777777777778,
"max": 0.9166666666666666,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/perturbations108": {
"correct": {
"median": 84,
"min": 84,
"max": 84,
"n": 3
},
"n": 108,
"acc": {
"median": 0.7777777777777778,
"min": 0.7777777777777778,
"max": 0.7777777777777778,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w1": {
"correct": {
"median": 29,
"min": 29,
"max": 29,
"n": 3
},
"n": 31,
"acc": {
"median": 0.9354838709677419,
"min": 0.9354838709677419,
"max": 0.9354838709677419,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/cicada-w2": {
"correct": {
"median": 19,
"min": 19,
"max": 19,
"n": 3
},
"n": 31,
"acc": {
"median": 0.6129032258064516,
"min": 0.6129032258064516,
"max": 0.6129032258064516,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd": {
"correct": {
"median": 65,
"min": 65,
"max": 65,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7738095238095238,
"min": 0.7738095238095238,
"max": 0.7738095238095238,
"n": 3
},
"blind_correct": {
"median": 50,
"min": 50,
"max": 50,
"n": 3
},
"controls": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit": {
"correct": {
"median": 10,
"min": 10,
"max": 10,
"n": 3
},
"n": 21,
"acc": {
"median": 0.47619047619047616,
"min": 0.47619047619047616,
"max": 0.47619047619047616,
"n": 3
},
"blind_correct": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"single/wyrd:exit2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/pooled": {
"correct": {
"median": 227,
"min": 227,
"max": 228,
"n": 3
},
"n": 259,
"acc": {
"median": 0.8764478764478765,
"min": 0.8764478764478765,
"max": 0.8803088803088803,
"n": 3
},
"blind_correct": {
"median": 65,
"min": 65,
"max": 65,
"n": 3
},
"controls": {
"median": 25,
"min": 25,
"max": 25,
"n": 3
},
"controls_n": 33,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/authored144": {
"correct": {
"median": 130,
"min": 130,
"max": 131,
"n": 3
},
"n": 144,
"acc": {
"median": 0.9027777777777778,
"min": 0.9027777777777778,
"max": 0.9097222222222222,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/perturbations108": {
"correct": {
"median": 87,
"min": 87,
"max": 87,
"n": 3
},
"n": 108,
"acc": {
"median": 0.8055555555555556,
"min": 0.8055555555555556,
"max": 0.8055555555555556,
"n": 3
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w1": {
"correct": {
"median": 30,
"min": 30,
"max": 30,
"n": 3
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/cicada-w2": {
"correct": {
"median": 18,
"min": 17,
"max": 18,
"n": 3
},
"n": 31,
"acc": {
"median": 0.5806451612903226,
"min": 0.5483870967741935,
"max": 0.5806451612903226,
"n": 3
},
"blind_correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"controls": {
"median": 4,
"min": 3,
"max": 4,
"n": 3
},
"controls_n": 5,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd": {
"correct": {
"median": 67,
"min": 67,
"max": 67,
"n": 3
},
"n": 84,
"acc": {
"median": 0.7976190476190477,
"min": 0.7976190476190477,
"max": 0.7976190476190477,
"n": 3
},
"blind_correct": {
"median": 50,
"min": 50,
"max": 50,
"n": 3
},
"controls": {
"median": 20,
"min": 20,
"max": 20,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place": {
"correct": {
"median": 18,
"min": 18,
"max": 18,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 11,
"min": 11,
"max": 11,
"n": 3
},
"n": 21,
"acc": {
"median": 0.5238095238095238,
"min": 0.5238095238095238,
"max": 0.5238095238095238,
"n": 3
},
"blind_correct": {
"median": 5,
"min": 5,
"max": 5,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/pooled": {
"correct": {
"median": 65.0,
"min": 65,
"max": 65,
"n": 2
},
"n": 84,
"acc": {
"median": 0.7738095238095238,
"min": 0.7738095238095238,
"max": 0.7738095238095238,
"n": 2
},
"blind_correct": null,
"controls": {
"median": 20.0,
"min": 20,
"max": 20,
"n": 2
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 2
}
},
"multifield/wyrd": {
"correct": {
"median": 65.0,
"min": 65,
"max": 65,
"n": 2
},
"n": 84,
"acc": {
"median": 0.7738095238095238,
"min": 0.7738095238095238,
"max": 0.7738095238095238,
"n": 2
},
"blind_correct": null,
"controls": {
"median": 20.0,
"min": 20,
"max": 20,
"n": 2
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 2
}
},
"multifield/wyrd:place": {
"correct": {
"median": 17.0,
"min": 17,
"max": 17,
"n": 2
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 2
},
"blind_correct": null,
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 2
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 2
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 2
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 2
},
"blind_correct": null,
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 2
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 2
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 10.0,
"min": 10,
"max": 10,
"n": 2
},
"n": 21,
"acc": {
"median": 0.47619047619047616,
"min": 0.47619047619047616,
"max": 0.47619047619047616,
"n": 2
},
"blind_correct": null,
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 2
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 2
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 17.0,
"min": 17,
"max": 17,
"n": 2
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 2
},
"blind_correct": null,
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 2
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 2
}
},
"agreement_signal": [
{
"unanimous": [
197,
217
],
"split": [
20,
35
]
},
{
"unanimous": [
197,
217
],
"split": [
20,
35
]
},
{
"unanimous": [
197,
216
],
"split": [
21,
36
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 2,
"flipped_ids": [
"ea7ec66a5e8eac886548",
"4533dd91267b4d815690"
],
"max_dp": 0.043490856885910034
},
{
"pair": "r2-r3",
"n": 560,
"flips": 2,
"flipped_ids": [
"ea7ec66a5e8eac886548",
"4533dd91267b4d815690"
],
"max_dp": 0.043490856885910034
}
],
"unstable_rows/single": {
"authored144": 2,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 2,
"repeats": 3
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 2,
"flipped_ids": [
"b531dd603bf080dab820",
"cicada-gate-w2-now/dog-died/gate"
],
"max_dp": 0.03331779843619964
},
{
"pair": "r2-r3",
"n": 560,
"flips": 2,
"flipped_ids": [
"b531dd603bf080dab820",
"cicada-gate-w2-now/dog-died/gate"
],
"max_dp": 0.03331779843619964
}
],
"unstable_rows/rotations": {
"authored144": 1,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 1,
"wyrd": 0,
"pooled": 1,
"repeats": 3
}
},
"order": {
"reversed": {
"label_changes": {
"median": 15,
"min": 15,
"max": 15,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.5383958667516708,
"min": 0.5383958667516708,
"max": 0.5559522360563278,
"n": 3
},
"median_row_max_dp": {
"median": 0.06593048572540283,
"min": 0.06593048572540283,
"max": 0.06741404347121716,
"n": 3
},
"acc": {
"median": 126,
"min": 126,
"max": 126,
"n": 3
}
},
"shuffled": {
"label_changes": {
"median": 10,
"min": 10,
"max": 10,
"n": 3
},
"n": 144,
"max_dp": {
"median": 0.43589483946561813,
"min": 0.43589483946561813,
"max": 0.4499364420771599,
"n": 3
},
"median_row_max_dp": {
"median": 0.06230974104255438,
"min": 0.06230974104255438,
"max": 0.06321389973163605,
"n": 3
},
"acc": {
"median": 131,
"min": 131,
"max": 131,
"n": 3
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 24,
"min": 24,
"max": 25,
"n": 3
},
"vs_original_gold": {
"median": 27,
"min": 27,
"max": 28,
"n": 3
},
"follows_description": {
"median": 106,
"min": 105,
"max": 106,
"n": 3
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
},
"peak_work_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
},
"peak_capacity_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 12064.0,
"min": 12064.0,
"max": 12064.0,
"n": 3
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 16.1,
"min": 16.1,
"max": 16.1,
"n": 3
},
"server_ms": {
"median": 14.7,
"min": 14.5,
"max": 14.7,
"n": 3
},
"run_medians_ms": [
15.9,
16.2
],
"tokens": 118,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 294.4,
"min": 291.5,
"max": 295.0,
"n": 3
},
"server_ms": {
"median": 292.5,
"min": 289.6,
"max": 293.1,
"n": 3
},
"run_medians_ms": [
290.8,
295.5
],
"tokens": 2489,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 214.6,
"min": 212.2,
"max": 215.4,
"n": 3
},
"server_ms": {
"median": 212.8,
"min": 210.2,
"max": 213.6,
"n": 3
},
"run_medians_ms": [
211.9,
215.7
],
"tokens": 3956,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 3377.7,
"min": 3374.5,
"max": 3383.7,
"n": 3
},
"server_ms": {
"median": 3375.7,
"min": 3372.4,
"max": 3381.8,
"n": 3
},
"run_medians_ms": [
3350.4,
3390.6
],
"tokens": 63302,
"non_200": 0
}
},
"capacity": {
"r1": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
200
],
[
128,
200
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
200
],
[
20,
200
],
[
24,
200
],
[
32,
200
],
[
48,
200
],
[
64,
200
]
]
}
},
"parity_vs_upstream_semif": [],
"paired_vs_semif": {
"single/pooled": {
"n": 259,
"semif": 217,
"cand": 224,
"fixed": 23,
"broken": 16,
"mcnemar_p": 0.3368,
"delta_pts": 2.7,
"delta_95ci_pts": [
-2.7,
7.9
]
},
"single/authored144": {
"n": 144,
"semif": 116,
"cand": 130,
"fixed": 22,
"broken": 8,
"mcnemar_p": 0.0161,
"delta_pts": 9.7,
"delta_95ci_pts": [
1.4,
18.1
]
},
"single/perturbations108": {
"n": 108,
"semif": 83,
"cand": 84,
"fixed": 10,
"broken": 9,
"mcnemar_p": 1.0,
"delta_pts": 0.9,
"delta_95ci_pts": [
-9.3,
11.1
]
},
"single/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"single/cicada-w2": {
"n": 31,
"semif": 20,
"cand": 19,
"fixed": 1,
"broken": 2,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-12.9,
6.5
]
},
"single/wyrd": {
"n": 84,
"semif": 71,
"cand": 65,
"fixed": 1,
"broken": 7,
"mcnemar_p": 0.0703,
"delta_pts": -7.1,
"delta_95ci_pts": [
-13.1,
-1.2
]
},
"single/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"single/wyrd:exit": {
"n": 21,
"semif": 16,
"cand": 10,
"fixed": 0,
"broken": 6,
"mcnemar_p": 0.0312,
"delta_pts": -28.6,
"delta_95ci_pts": [
-47.6,
-9.5
]
},
"rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 227,
"fixed": 10,
"broken": 15,
"mcnemar_p": 0.4244,
"delta_pts": -1.9,
"delta_95ci_pts": [
-5.4,
1.5
]
},
"rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 130,
"fixed": 7,
"broken": 8,
"mcnemar_p": 1.0,
"delta_pts": -0.7,
"delta_95ci_pts": [
-5.6,
3.5
]
},
"rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 87,
"fixed": 4,
"broken": 10,
"mcnemar_p": 0.1796,
"delta_pts": -5.6,
"delta_95ci_pts": [
-15.7,
3.7
]
},
"rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 30,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 18,
"fixed": 1,
"broken": 2,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-16.1,
6.5
]
},
"rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 67,
"fixed": 3,
"broken": 7,
"mcnemar_p": 0.3438,
"delta_pts": -4.8,
"delta_95ci_pts": [
-11.9,
2.4
]
},
"rotations/wyrd:place2": {
"n": 21,
"semif": 21,
"cand": 21,
"fixed": 0,
"broken": 0,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
0.0,
0.0
]
},
"rotations/wyrd:exit": {
"n": 21,
"semif": 18,
"cand": 11,
"fixed": 0,
"broken": 7,
"mcnemar_p": 0.0156,
"delta_pts": -33.3,
"delta_95ci_pts": [
-52.4,
-14.3
]
},
"cand-single-vs-semif-rotations/pooled": {
"n": 259,
"semif": 232,
"cand": 224,
"fixed": 10,
"broken": 18,
"mcnemar_p": 0.1849,
"delta_pts": -3.1,
"delta_95ci_pts": [
-7.3,
0.8
]
},
"cand-single-vs-semif-rotations/authored144": {
"n": 144,
"semif": 131,
"cand": 130,
"fixed": 7,
"broken": 8,
"mcnemar_p": 1.0,
"delta_pts": -0.7,
"delta_95ci_pts": [
-6.2,
4.2
]
},
"cand-single-vs-semif-rotations/perturbations108": {
"n": 108,
"semif": 93,
"cand": 84,
"fixed": 3,
"broken": 12,
"mcnemar_p": 0.0352,
"delta_pts": -8.3,
"delta_95ci_pts": [
-17.6,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w1": {
"n": 31,
"semif": 30,
"cand": 29,
"fixed": 0,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": -3.2,
"delta_95ci_pts": [
-9.7,
0.0
]
},
"cand-single-vs-semif-rotations/cicada-w2": {
"n": 31,
"semif": 19,
"cand": 19,
"fixed": 1,
"broken": 1,
"mcnemar_p": 1.0,
"delta_pts": 0.0,
"delta_95ci_pts": [
-9.7,
9.7
]
},
"cand-single-vs-semif-rotations/wyrd": {
"n": 84,
"semif": 71,
"cand": 65,
"fixed": 3,
"broken": 9,
"mcnemar_p": 0.146,
"delta_pts": -7.1,
"delta_95ci_pts": [
-15.5,
1.2
]
}
}
},
"semif-qwen35-4b": {
"repeats": [
"r1",
"r2",
"r3",
"r4"
],
"jevbench": {
"easy": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"easy_n": {
"median": 48.0,
"min": 48,
"max": 48,
"n": 4
},
"original": {
"median": 0.9861111111111112,
"min": 0.9861111111111112,
"max": 0.9861111111111112,
"n": 4
},
"original_n": {
"median": 71.0,
"min": 71,
"max": 71,
"n": 4
},
"hard": {
"median": 0.6126126126126126,
"min": 0.6126126126126126,
"max": 0.6126126126126126,
"n": 4
},
"hard_n": {
"median": 68.0,
"min": 68,
"max": 68,
"n": 4
},
"all": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 4
},
"all_n": {
"median": 187.0,
"min": 187,
"max": 187,
"n": 4
},
"failed": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
},
"p50_ms": {
"median": 35.95828462857753,
"min": 35.729505121707916,
"max": 36.197722889482975,
"n": 4
}
},
"sets": {
"single/pooled": {
"correct": {
"median": 217.0,
"min": 217,
"max": 217,
"n": 4
},
"n": 259,
"acc": {
"median": 0.8378378378378378,
"min": 0.8378378378378378,
"max": 0.8378378378378378,
"n": 4
},
"blind_correct": {
"median": 76.0,
"min": 76,
"max": 76,
"n": 4
},
"controls": {
"median": 26.0,
"min": 26,
"max": 26,
"n": 4
},
"controls_n": 33,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/authored144": {
"correct": {
"median": 116.0,
"min": 116,
"max": 116,
"n": 4
},
"n": 144,
"acc": {
"median": 0.8055555555555556,
"min": 0.8055555555555556,
"max": 0.8055555555555556,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/perturbations108": {
"correct": {
"median": 83.0,
"min": 83,
"max": 83,
"n": 4
},
"n": 108,
"acc": {
"median": 0.7685185185185185,
"min": 0.7685185185185185,
"max": 0.7685185185185185,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/cicada-w1": {
"correct": {
"median": 30.0,
"min": 30,
"max": 30,
"n": 4
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/cicada-w2": {
"correct": {
"median": 20.0,
"min": 20,
"max": 20,
"n": 4
},
"n": 31,
"acc": {
"median": 0.6451612903225806,
"min": 0.6451612903225806,
"max": 0.6451612903225806,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd": {
"correct": {
"median": 71.0,
"min": 71,
"max": 71,
"n": 4
},
"n": 84,
"acc": {
"median": 0.8452380952380952,
"min": 0.8452380952380952,
"max": 0.8452380952380952,
"n": 4
},
"blind_correct": {
"median": 60.0,
"min": 60,
"max": 60,
"n": 4
},
"controls": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 4
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:place": {
"correct": {
"median": 17.0,
"min": 17,
"max": 17,
"n": 4
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:place2": {
"correct": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 4
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:exit": {
"correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"single/wyrd:exit2": {
"correct": {
"median": 17.0,
"min": 17,
"max": 17,
"n": 4
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/pooled": {
"correct": {
"median": 232.0,
"min": 232,
"max": 232,
"n": 4
},
"n": 259,
"acc": {
"median": 0.8957528957528957,
"min": 0.8957528957528957,
"max": 0.8957528957528957,
"n": 4
},
"blind_correct": {
"median": 76.0,
"min": 76,
"max": 76,
"n": 4
},
"controls": {
"median": 26.0,
"min": 26,
"max": 26,
"n": 4
},
"controls_n": 33,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/authored144": {
"correct": {
"median": 131.0,
"min": 131,
"max": 131,
"n": 4
},
"n": 144,
"acc": {
"median": 0.9097222222222222,
"min": 0.9097222222222222,
"max": 0.9097222222222222,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/perturbations108": {
"correct": {
"median": 93.0,
"min": 93,
"max": 93,
"n": 4
},
"n": 108,
"acc": {
"median": 0.8611111111111112,
"min": 0.8611111111111112,
"max": 0.8611111111111112,
"n": 4
},
"blind_correct": null,
"controls": null,
"controls_n": 0,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/cicada-w1": {
"correct": {
"median": 30.0,
"min": 30,
"max": 30,
"n": 4
},
"n": 31,
"acc": {
"median": 0.967741935483871,
"min": 0.967741935483871,
"max": 0.967741935483871,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 5.0,
"min": 5,
"max": 5,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/cicada-w2": {
"correct": {
"median": 19.0,
"min": 19,
"max": 19,
"n": 4
},
"n": 31,
"acc": {
"median": 0.6129032258064516,
"min": 0.6129032258064516,
"max": 0.6129032258064516,
"n": 4
},
"blind_correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 5,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd": {
"correct": {
"median": 71.0,
"min": 71,
"max": 71,
"n": 4
},
"n": 84,
"acc": {
"median": 0.8452380952380952,
"min": 0.8452380952380952,
"max": 0.8452380952380952,
"n": 4
},
"blind_correct": {
"median": 60.0,
"min": 60,
"max": 60,
"n": 4
},
"controls": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 4
},
"controls_n": 28,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:place": {
"correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:place2": {
"correct": {
"median": 21.0,
"min": 21,
"max": 21,
"n": 4
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 7.0,
"min": 7,
"max": 7,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:exit": {
"correct": {
"median": 18.0,
"min": 18,
"max": 18,
"n": 4
},
"n": 21,
"acc": {
"median": 0.8571428571428571,
"min": 0.8571428571428571,
"max": 0.8571428571428571,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 6.0,
"min": 6,
"max": 6,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"rotations/wyrd:exit2": {
"correct": {
"median": 16.0,
"min": 16,
"max": 16,
"n": 4
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 4
},
"blind_correct": {
"median": 15.0,
"min": 15,
"max": 15,
"n": 4
},
"controls": {
"median": 4.0,
"min": 4,
"max": 4,
"n": 4
},
"controls_n": 7,
"failures": {
"median": 0.0,
"min": 0,
"max": 0,
"n": 4
}
},
"multifield/pooled": {
"correct": {
"median": 71,
"min": 71,
"max": 71,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8452380952380952,
"min": 0.8452380952380952,
"max": 0.8452380952380952,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd": {
"correct": {
"median": 71,
"min": 71,
"max": 71,
"n": 3
},
"n": 84,
"acc": {
"median": 0.8452380952380952,
"min": 0.8452380952380952,
"max": 0.8452380952380952,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"controls_n": 28,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:place2": {
"correct": {
"median": 21,
"min": 21,
"max": 21,
"n": 3
},
"n": 21,
"acc": {
"median": 1.0,
"min": 1.0,
"max": 1.0,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 7,
"min": 7,
"max": 7,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit": {
"correct": {
"median": 16,
"min": 16,
"max": 16,
"n": 3
},
"n": 21,
"acc": {
"median": 0.7619047619047619,
"min": 0.7619047619047619,
"max": 0.7619047619047619,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 6,
"min": 6,
"max": 6,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"multifield/wyrd:exit2": {
"correct": {
"median": 17,
"min": 17,
"max": 17,
"n": 3
},
"n": 21,
"acc": {
"median": 0.8095238095238095,
"min": 0.8095238095238095,
"max": 0.8095238095238095,
"n": 3
},
"blind_correct": null,
"controls": {
"median": 4,
"min": 4,
"max": 4,
"n": 3
},
"controls_n": 7,
"failures": {
"median": 0,
"min": 0,
"max": 0,
"n": 3
}
},
"agreement_signal": [
{
"unanimous": [
153,
162
],
"split": [
71,
90
]
},
{
"unanimous": [
153,
162
],
"split": [
71,
90
]
},
{
"unanimous": [
153,
162
],
"split": [
71,
90
]
},
{
"unanimous": [
153,
162
],
"split": [
71,
90
]
}
]
},
"floors": {
"a_vs_a_in_process": [
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
},
{
"flips": 0,
"max_dp": 0.0,
"n": 144
}
],
"cross_restart/single": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r3-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/single": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 4
},
"cross_restart/rotations": [
{
"pair": "r1-r2",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r1-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r3",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r2-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
},
{
"pair": "r3-r4",
"n": 560,
"flips": 0,
"flipped_ids": [],
"max_dp": 0.0
}
],
"unstable_rows/rotations": {
"authored144": 0,
"perturbations108": 0,
"cicada-w1": 0,
"cicada-w2": 0,
"wyrd": 0,
"pooled": 0,
"repeats": 4
}
},
"order": {
"reversed": {
"label_changes": {
"median": 30.0,
"min": 30,
"max": 30,
"n": 4
},
"n": 144,
"max_dp": {
"median": 0.8944366923597074,
"min": 0.8944366923597074,
"max": 0.8944366923597074,
"n": 4
},
"median_row_max_dp": {
"median": 0.09456913033051326,
"min": 0.09456913033051326,
"max": 0.09456913033051326,
"n": 4
},
"acc": {
"median": 122.0,
"min": 122,
"max": 122,
"n": 4
}
},
"shuffled": {
"label_changes": {
"median": 29.0,
"min": 29,
"max": 29,
"n": 4
},
"n": 144,
"max_dp": {
"median": 0.8944366923597074,
"min": 0.8944366923597074,
"max": 0.8944366923597074,
"n": 4
},
"median_row_max_dp": {
"median": 0.0733941869533889,
"min": 0.0733941869533889,
"max": 0.0733941869533889,
"n": 4
},
"acc": {
"median": 121.0,
"min": 121,
"max": 121,
"n": 4
}
}
},
"negative": {
"same_top_as_unrotated": {
"median": 14.0,
"min": 14,
"max": 14,
"n": 4
},
"vs_original_gold": {
"median": 17.0,
"min": 17,
"max": 17,
"n": 4
},
"follows_description": {
"median": 119.0,
"min": 119,
"max": 119,
"n": 4
},
"n": 144
},
"vram": {
"rest_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9250.0,
"n": 4
},
"peak_work_mib": {
"median": 12918.0,
"min": 12886.0,
"max": 12918.0,
"n": 4
},
"peak_capacity_mib": {
"median": 12920.0,
"min": 12920.0,
"max": 12920.0,
"n": 1
},
"peak_jevbench_mib": {
"median": 9242.0,
"min": 9242.0,
"max": 9242.0,
"n": 4
}
},
"latency": {
"short1": {
"e2e_ms": {
"median": 37.349999999999994,
"min": 37.2,
"max": 37.7,
"n": 4
},
"server_ms": {
"median": 34.55,
"min": 34.3,
"max": 34.9,
"n": 4
},
"run_medians_ms": [
37.1,
37.7
],
"tokens": 114,
"non_200": 0
},
"crit21": {
"e2e_ms": {
"median": 131.10000000000002,
"min": 130.9,
"max": 131.7,
"n": 4
},
"server_ms": {
"median": 119.35,
"min": 119.1,
"max": 119.5,
"n": 4
},
"run_medians_ms": [
130.4,
132.6
],
"tokens": 62,
"non_200": 0
},
"long1": {
"e2e_ms": {
"median": 185.7,
"min": 185.6,
"max": 187.4,
"n": 4
},
"server_ms": {
"median": 182.15,
"min": 182.0,
"max": 183.6,
"n": 4
},
"run_medians_ms": [
183.9,
187.8
],
"tokens": 3952,
"non_200": 0
},
"long16": {
"e2e_ms": {
"median": 505.05,
"min": 501.5,
"max": 519.3,
"n": 4
},
"server_ms": {
"median": 486.20000000000005,
"min": 482.5,
"max": 499.9,
"n": 4
},
"run_medians_ms": [
501.1,
520.0
],
"tokens": 3900,
"non_200": 0
}
},
"capacity": {
"r1": {
"short": [
[
16,
200
],
[
32,
200
],
[
48,
200
],
[
56,
200
],
[
60,
200
],
[
63,
200
],
[
64,
200
],
[
96,
422
]
],
"long": [
[
1,
200
],
[
4,
200
],
[
8,
200
],
[
12,
200
],
[
14,
200
],
[
16,
200
],
[
18,
503
]
]
}
},
"parity_vs_upstream_semif": [
{
"n": 144,
"top_agree": 144,
"prompt_sha_equal": 144,
"max_dp": 0.0595381559570336
},
{
"n": 144,
"top_agree": 144,
"prompt_sha_equal": 144,
"max_dp": 0.0595381559570336
},
{
"n": 144,
"top_agree": 144,
"prompt_sha_equal": 144,
"max_dp": 0.0595381559570336
},
{
"n": 144,
"top_agree": 144,
"prompt_sha_equal": 144,
"max_dp": 0.0595381559570336
}
]
}
}
Reference in New Issue View Git Blame Copy Permalink