vh
1cf763a7b1
intern-decision deployed 0941 PT (cap 9.0 GiB, MAX_TOKENS 7168). Live acceptance: positive control 240/259 and Wyrd 79/84, bit-identical to the bench (0/560 rows, Δp 0); negative control 10/122/14; largest accepted requests 200 with no 503; per-process 8,812 MiB at rest and 9,866 peak; GPU 1 Free 15,442 before and 6,581 after (lowest 5,569 under load). Latency from nh3-dev: 21 criteria 114 ms, 16 over ~3,900 tokens 238 ms. semif README banner now REPLACED with the rollback; fv-ml1 GPU 1 note updated.
672 lines
8.9 KiB
JSON
672 lines
8.9 KiB
JSON
{
|
|
"ours": {
|
|
"live": {
|
|
"scores": {
|
|
"single/pooled": [
|
|
240,
|
|
259
|
|
],
|
|
"single/authored144": [
|
|
132,
|
|
144
|
|
],
|
|
"single/perturbations108": [
|
|
100,
|
|
108
|
|
],
|
|
"single/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"single/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"single/wyrd": [
|
|
79,
|
|
84
|
|
],
|
|
"single/wyrd:place": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"single/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"single/failures": 0
|
|
},
|
|
"negative": {
|
|
"n": 144,
|
|
"same_top_as_unrotated": 10,
|
|
"follows_description": 122,
|
|
"vs_original_gold": 14
|
|
},
|
|
"a_vs_a_in_process_authored144": {
|
|
"rows": 0,
|
|
"top_differs": 0,
|
|
"flipped_ids": [],
|
|
"max_dp": null
|
|
}
|
|
}
|
|
},
|
|
"bench": {
|
|
"r1": {
|
|
"scores": {
|
|
"single/pooled": [
|
|
240,
|
|
259
|
|
],
|
|
"single/authored144": [
|
|
132,
|
|
144
|
|
],
|
|
"single/perturbations108": [
|
|
100,
|
|
108
|
|
],
|
|
"single/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"single/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"single/wyrd": [
|
|
79,
|
|
84
|
|
],
|
|
"single/wyrd:place": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"single/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"single/failures": 0,
|
|
"rotations/pooled": [
|
|
236,
|
|
259
|
|
],
|
|
"rotations/authored144": [
|
|
130,
|
|
144
|
|
],
|
|
"rotations/perturbations108": [
|
|
99,
|
|
108
|
|
],
|
|
"rotations/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"rotations/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"rotations/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"rotations/wyrd:place": [
|
|
18,
|
|
21
|
|
],
|
|
"rotations/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit2": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/failures": 0
|
|
},
|
|
"negative": {
|
|
"n": 144,
|
|
"same_top_as_unrotated": 10,
|
|
"follows_description": 122,
|
|
"vs_original_gold": 14
|
|
}
|
|
},
|
|
"r2": {
|
|
"scores": {
|
|
"single/pooled": [
|
|
240,
|
|
259
|
|
],
|
|
"single/authored144": [
|
|
132,
|
|
144
|
|
],
|
|
"single/perturbations108": [
|
|
100,
|
|
108
|
|
],
|
|
"single/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"single/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"single/wyrd": [
|
|
79,
|
|
84
|
|
],
|
|
"single/wyrd:place": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"single/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"single/failures": 0,
|
|
"rotations/pooled": [
|
|
236,
|
|
259
|
|
],
|
|
"rotations/authored144": [
|
|
130,
|
|
144
|
|
],
|
|
"rotations/perturbations108": [
|
|
99,
|
|
108
|
|
],
|
|
"rotations/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"rotations/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"rotations/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"rotations/wyrd:place": [
|
|
18,
|
|
21
|
|
],
|
|
"rotations/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit2": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/failures": 0,
|
|
"multifield/pooled": [
|
|
77,
|
|
84
|
|
],
|
|
"multifield/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"multifield/wyrd:place": [
|
|
16,
|
|
21
|
|
],
|
|
"multifield/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"multifield/wyrd:exit": [
|
|
20,
|
|
21
|
|
],
|
|
"multifield/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"multifield/failures": 0
|
|
},
|
|
"negative": {
|
|
"n": 144,
|
|
"same_top_as_unrotated": 10,
|
|
"follows_description": 122,
|
|
"vs_original_gold": 14
|
|
}
|
|
},
|
|
"r3": {
|
|
"scores": {
|
|
"single/pooled": [
|
|
240,
|
|
259
|
|
],
|
|
"single/authored144": [
|
|
132,
|
|
144
|
|
],
|
|
"single/perturbations108": [
|
|
100,
|
|
108
|
|
],
|
|
"single/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"single/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"single/wyrd": [
|
|
79,
|
|
84
|
|
],
|
|
"single/wyrd:place": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"single/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"single/failures": 0,
|
|
"rotations/pooled": [
|
|
236,
|
|
259
|
|
],
|
|
"rotations/authored144": [
|
|
130,
|
|
144
|
|
],
|
|
"rotations/perturbations108": [
|
|
99,
|
|
108
|
|
],
|
|
"rotations/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"rotations/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"rotations/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"rotations/wyrd:place": [
|
|
18,
|
|
21
|
|
],
|
|
"rotations/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit2": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/failures": 0,
|
|
"multifield/pooled": [
|
|
77,
|
|
84
|
|
],
|
|
"multifield/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"multifield/wyrd:place": [
|
|
16,
|
|
21
|
|
],
|
|
"multifield/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"multifield/wyrd:exit": [
|
|
20,
|
|
21
|
|
],
|
|
"multifield/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"multifield/failures": 0
|
|
},
|
|
"negative": {
|
|
"n": 144,
|
|
"same_top_as_unrotated": 10,
|
|
"follows_description": 122,
|
|
"vs_original_gold": 14
|
|
}
|
|
},
|
|
"r4": {
|
|
"scores": {
|
|
"single/pooled": [
|
|
240,
|
|
259
|
|
],
|
|
"single/authored144": [
|
|
132,
|
|
144
|
|
],
|
|
"single/perturbations108": [
|
|
100,
|
|
108
|
|
],
|
|
"single/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"single/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"single/wyrd": [
|
|
79,
|
|
84
|
|
],
|
|
"single/wyrd:place": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"single/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"single/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"single/failures": 0,
|
|
"rotations/pooled": [
|
|
236,
|
|
259
|
|
],
|
|
"rotations/authored144": [
|
|
130,
|
|
144
|
|
],
|
|
"rotations/perturbations108": [
|
|
99,
|
|
108
|
|
],
|
|
"rotations/cicada-w1": [
|
|
29,
|
|
31
|
|
],
|
|
"rotations/cicada-w2": [
|
|
22,
|
|
31
|
|
],
|
|
"rotations/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"rotations/wyrd:place": [
|
|
18,
|
|
21
|
|
],
|
|
"rotations/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/wyrd:exit2": [
|
|
19,
|
|
21
|
|
],
|
|
"rotations/failures": 0,
|
|
"multifield/pooled": [
|
|
77,
|
|
84
|
|
],
|
|
"multifield/wyrd": [
|
|
77,
|
|
84
|
|
],
|
|
"multifield/wyrd:place": [
|
|
16,
|
|
21
|
|
],
|
|
"multifield/wyrd:place2": [
|
|
21,
|
|
21
|
|
],
|
|
"multifield/wyrd:exit": [
|
|
20,
|
|
21
|
|
],
|
|
"multifield/wyrd:exit2": [
|
|
20,
|
|
21
|
|
],
|
|
"multifield/failures": 0
|
|
},
|
|
"negative": {
|
|
"n": 144,
|
|
"same_top_as_unrotated": 10,
|
|
"follows_description": 122,
|
|
"vs_original_gold": 14
|
|
}
|
|
}
|
|
},
|
|
"vs_bench_row_by_row": {
|
|
"live": {
|
|
"single": {
|
|
"bench_repeat": "r1",
|
|
"rows": 560,
|
|
"top_differs": 0,
|
|
"flipped_ids": [],
|
|
"max_dp": 0.0
|
|
},
|
|
"rotations": {
|
|
"bench_repeat": "r1",
|
|
"rows": 0,
|
|
"top_differs": 0,
|
|
"flipped_ids": [],
|
|
"max_dp": null
|
|
},
|
|
"negative": {
|
|
"bench_repeat": "r1",
|
|
"rows": 144,
|
|
"top_differs": 0,
|
|
"flipped_ids": [],
|
|
"max_dp": 0.0
|
|
},
|
|
"multifield": {
|
|
"bench_repeat": "r2",
|
|
"rows": 0,
|
|
"top_differs": 0,
|
|
"flipped_ids": [],
|
|
"max_dp": null
|
|
}
|
|
}
|
|
},
|
|
"across_restarts": {},
|
|
"summary": {
|
|
"single/authored144": {
|
|
"ours_median": 132,
|
|
"ours_all": [
|
|
132
|
|
],
|
|
"bench_all": [
|
|
132,
|
|
132,
|
|
132,
|
|
132
|
|
],
|
|
"n": 144
|
|
},
|
|
"single/cicada-w1": {
|
|
"ours_median": 29,
|
|
"ours_all": [
|
|
29
|
|
],
|
|
"bench_all": [
|
|
29,
|
|
29,
|
|
29,
|
|
29
|
|
],
|
|
"n": 31
|
|
},
|
|
"single/cicada-w2": {
|
|
"ours_median": 22,
|
|
"ours_all": [
|
|
22
|
|
],
|
|
"bench_all": [
|
|
22,
|
|
22,
|
|
22,
|
|
22
|
|
],
|
|
"n": 31
|
|
},
|
|
"single/perturbations108": {
|
|
"ours_median": 100,
|
|
"ours_all": [
|
|
100
|
|
],
|
|
"bench_all": [
|
|
100,
|
|
100,
|
|
100,
|
|
100
|
|
],
|
|
"n": 108
|
|
},
|
|
"single/pooled": {
|
|
"ours_median": 240,
|
|
"ours_all": [
|
|
240
|
|
],
|
|
"bench_all": [
|
|
240,
|
|
240,
|
|
240,
|
|
240
|
|
],
|
|
"n": 259
|
|
},
|
|
"single/wyrd": {
|
|
"ours_median": 79,
|
|
"ours_all": [
|
|
79
|
|
],
|
|
"bench_all": [
|
|
79,
|
|
79,
|
|
79,
|
|
79
|
|
],
|
|
"n": 84
|
|
},
|
|
"single/wyrd:exit": {
|
|
"ours_median": 19,
|
|
"ours_all": [
|
|
19
|
|
],
|
|
"bench_all": [
|
|
19,
|
|
19,
|
|
19,
|
|
19
|
|
],
|
|
"n": 21
|
|
},
|
|
"single/wyrd:exit2": {
|
|
"ours_median": 20,
|
|
"ours_all": [
|
|
20
|
|
],
|
|
"bench_all": [
|
|
20,
|
|
20,
|
|
20,
|
|
20
|
|
],
|
|
"n": 21
|
|
},
|
|
"single/wyrd:place": {
|
|
"ours_median": 19,
|
|
"ours_all": [
|
|
19
|
|
],
|
|
"bench_all": [
|
|
19,
|
|
19,
|
|
19,
|
|
19
|
|
],
|
|
"n": 21
|
|
},
|
|
"single/wyrd:place2": {
|
|
"ours_median": 21,
|
|
"ours_all": [
|
|
21
|
|
],
|
|
"bench_all": [
|
|
21,
|
|
21,
|
|
21,
|
|
21
|
|
],
|
|
"n": 21
|
|
}
|
|
}
|
|
}
|