vh
ff552abf1f
Straight passthrough to the checkpoint's own DecisionEngine.predict — never the semif mapping, whose different prompt would change the answers. Reuses the one inference thread, bearer auth, MAX_QUEUE, VRAM cap and error envelope; no new concurrency. 1..16 questions in ONE call (never chunked: Jev questions share a prompt); images 422; over MAX_TOKENS 422 before the forward. /health advertises the surface. Response 'model' is a string name@revision (JevBench's runner hashes it; a dict broke its manifest step). Acceptance on the live service (see acceptance/systemone-2026-09-30/): JevBench v1.2.16 typesafe adapter over the 231 public items scores all 202/231, hard 83/111, with 0 changed answers across all 924 rows of the bench's own r1..r4; controls 401/422x3 (token boundary proven at 7168 pass / 7169 refuse); GPU 1 per-process peak 9,866 MiB under the largest accepted request (budget 9,876); /decide/shared positive control unchanged. 120 tests green.
2142 lines
49 KiB
JSON
2142 lines
49 KiB
JSON
{
|
|
"accuracy": 0.8744588744588745,
|
|
"brier_mean": 0.17806609620099598,
|
|
"calibration_n": 231,
|
|
"complete": true,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.4,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.3623595989308779,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.46124870941679735,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5637253221582417,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.65,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.648800445448587,
|
|
"n": 20
|
|
},
|
|
{
|
|
"accuracy": 0.7391304347826086,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7523100109863671,
|
|
"n": 23
|
|
},
|
|
{
|
|
"accuracy": 0.9117647058823529,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8641129171311692,
|
|
"n": 34
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9627515382033693,
|
|
"n": 129
|
|
}
|
|
],
|
|
"ece": 0.051072415884039386,
|
|
"n": 231
|
|
},
|
|
"latency": {
|
|
"n": 231,
|
|
"p50_s": 0.054981209977995604,
|
|
"p95_s": 0.14865703746909276
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"ledger_charged_usd": 0.0,
|
|
"macro_accuracy": 0.8725744098809114,
|
|
"model_identities": [
|
|
"Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd"
|
|
],
|
|
"n_attempted": 231,
|
|
"n_correct": 202,
|
|
"n_planned": 231,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 231,
|
|
"n_valid": 231,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.26234367203481096,
|
|
"paraphrase_consistency": {
|
|
"agree": 35,
|
|
"agreement": 0.9722222222222222,
|
|
"both_correct_rate_all_pairs": 0.9722222222222222,
|
|
"both_valid": 36,
|
|
"pairs": 36
|
|
},
|
|
"per_family": {
|
|
"adequacy": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.04603689738961828,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5930110454411601,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7557061813028185,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8878357373215198,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9520119763886462,
|
|
"n": 8
|
|
}
|
|
],
|
|
"ece": 0.104959623958651,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.05385151947848499,
|
|
"p95_s": 0.05537226824963
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"adversarial": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.011617825043304442,
|
|
"calibration_n": 6,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8627679157764538,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9629914990167525,
|
|
"n": 4
|
|
}
|
|
],
|
|
"ece": 0.07041636206334703,
|
|
"n": 6
|
|
},
|
|
"latency": {
|
|
"n": 6,
|
|
"p50_s": 0.05564080100157298,
|
|
"p95_s": 0.058122831251239404
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 6,
|
|
"n_correct": 6,
|
|
"n_planned": 6,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 6,
|
|
"n_valid": 6,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.10376329334202228,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"ambiguous": {
|
|
"accuracy": 0.7142857142857143,
|
|
"brier_mean": 0.3223035707740803,
|
|
"calibration_n": 7,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.3333333333333333,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6479833643674017,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9463672181616865,
|
|
"n": 4
|
|
}
|
|
],
|
|
"ece": 0.16549731720792277,
|
|
"n": 7
|
|
},
|
|
"latency": {
|
|
"n": 7,
|
|
"p50_s": 0.06240525998873636,
|
|
"p95_s": 0.06425336602260359
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 7,
|
|
"n_correct": 5,
|
|
"n_planned": 7,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 7,
|
|
"n_valid": 7,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"extraction": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.0019479746093977082,
|
|
"calibration_n": 24,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9683153887441366,
|
|
"n": 24
|
|
}
|
|
],
|
|
"ece": 0.031684611255863415,
|
|
"n": 24
|
|
},
|
|
"latency": {
|
|
"n": 24,
|
|
"p50_s": 0.05352575849974528,
|
|
"p95_s": 0.05614419543999247
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 24,
|
|
"n_correct": 24,
|
|
"n_planned": 24,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 24,
|
|
"n_valid": 24,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"fact": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.00253357674402684,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9707540110368739,
|
|
"n": 12
|
|
}
|
|
],
|
|
"ece": 0.029245988963126135,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.052916569489752874,
|
|
"p95_s": 0.053868782028439456
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"intent": {
|
|
"accuracy": 0.9583333333333334,
|
|
"brier_mean": 0.04236379090188356,
|
|
"calibration_n": 24,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4791247461286767,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.697318944283325,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7504260155024499,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8623501745620707,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.97124439087109,
|
|
"n": 17
|
|
}
|
|
],
|
|
"ece": 0.0909478584310712,
|
|
"n": 24
|
|
},
|
|
"latency": {
|
|
"n": 24,
|
|
"p50_s": 0.05407417201786302,
|
|
"p95_s": 0.059262538660550486
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 24,
|
|
"n_correct": 23,
|
|
"n_planned": 24,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 24,
|
|
"n_valid": 24,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"judge_hard": {
|
|
"accuracy": 0.9411764705882353,
|
|
"brier_mean": 0.1769820408934362,
|
|
"calibration_n": 17,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5469149563843179,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6659927647326704,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7429117727934124,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8717155645981629,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9748315019610428,
|
|
"n": 5
|
|
}
|
|
],
|
|
"ece": 0.15040904282950293,
|
|
"n": 17
|
|
},
|
|
"latency": {
|
|
"n": 17,
|
|
"p50_s": 0.054450018040370196,
|
|
"p95_s": 0.055332889617420736
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 17,
|
|
"n_correct": 16,
|
|
"n_planned": 17,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 17,
|
|
"n_valid": 17,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"long_policy": {
|
|
"accuracy": 0.6842105263157895,
|
|
"brier_mean": 0.488183253400819,
|
|
"calibration_n": 19,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.32987837012316157,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4247953847281636,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5930110454411601,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 0.6666666666666666,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6374422769566587,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7434209921956753,
|
|
"n": 4
|
|
},
|
|
{
|
|
"accuracy": 0.8333333333333334,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8591073374831182,
|
|
"n": 6
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9264976568409564,
|
|
"n": 2
|
|
}
|
|
],
|
|
"ece": 0.13342281306158954,
|
|
"n": 19
|
|
},
|
|
"latency": {
|
|
"n": 19,
|
|
"p50_s": 0.15093205898301676,
|
|
"p95_s": 0.8460185076109925
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 19,
|
|
"n_correct": 13,
|
|
"n_planned": 19,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 19,
|
|
"n_valid": 19,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.8438052410179581,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"multi_hop": {
|
|
"accuracy": 0.8333333333333334,
|
|
"brier_mean": 0.2821094188229273,
|
|
"calibration_n": 18,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.3946907024781775,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.75,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5453134547809111,
|
|
"n": 4
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6537803635138819,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7674221407596936,
|
|
"n": 4
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8512239723447685,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9325685206539565,
|
|
"n": 2
|
|
}
|
|
],
|
|
"ece": 0.22582924507652405,
|
|
"n": 18
|
|
},
|
|
"latency": {
|
|
"n": 18,
|
|
"p50_s": 0.14022018198738806,
|
|
"p95_s": 0.1504995027236873
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 18,
|
|
"n_correct": 15,
|
|
"n_planned": 18,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 18,
|
|
"n_valid": 18,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.3268150682115487,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"ordinal": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.006310346320679678,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8500853530913438,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.956899632823367,
|
|
"n": 11
|
|
}
|
|
],
|
|
"ece": 0.05200155715430161,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.054876862006494775,
|
|
"p95_s": 0.06036159305367618
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.0531333548567569,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"policy": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.020422084182937634,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7557061813028185,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8742406957013814,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9513615324190499,
|
|
"n": 9
|
|
}
|
|
],
|
|
"ece": 0.07779655296024748,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.05428065449814312,
|
|
"p95_s": 0.05681873176072258
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"probability": {
|
|
"accuracy": 0.7,
|
|
"brier_mean": 0.46568152828081544,
|
|
"calibration_n": 10,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5777854893190272,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6325445353204342,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.3333333333333333,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7561351202525173,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8908360355378411,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
}
|
|
],
|
|
"ece": 0.26370611592985127,
|
|
"n": 10
|
|
},
|
|
"latency": {
|
|
"n": 10,
|
|
"p50_s": 0.07672692849882878,
|
|
"p95_s": 0.09677280441683249
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 10,
|
|
"n_correct": 7,
|
|
"n_planned": 10,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 10,
|
|
"n_valid": 10,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"routing": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.041188901489954015,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6801954980304856,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7877520218832728,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8825011701644436,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9364805526689252,
|
|
"n": 7
|
|
}
|
|
],
|
|
"ece": 0.12762423108703272,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.05457381851738319,
|
|
"p95_s": 0.05613349811465014
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"routing_hard": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.0051631143261297256,
|
|
"calibration_n": 5,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9423656177532307,
|
|
"n": 5
|
|
}
|
|
],
|
|
"ece": 0.05763438224676931,
|
|
"n": 5
|
|
},
|
|
"latency": {
|
|
"n": 5,
|
|
"p50_s": 0.055770282982848585,
|
|
"p95_s": 0.05623367396183312
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 5,
|
|
"n_correct": 5,
|
|
"n_planned": 5,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 5,
|
|
"n_valid": 5,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"temporal_numeric": {
|
|
"accuracy": 0.3333333333333333,
|
|
"brier_mean": 0.7767759878778288,
|
|
"calibration_n": 15,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.2857142857142857,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.36702122094061124,
|
|
"n": 7
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4681313198099223,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5726384329365539,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6472762711048763,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7318060856600669,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8512242138235586,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
}
|
|
],
|
|
"ece": 0.2179071110112772,
|
|
"n": 15
|
|
},
|
|
"latency": {
|
|
"n": 15,
|
|
"p50_s": 0.06412995798746124,
|
|
"p95_s": 0.07775642202468588
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 15,
|
|
"n_correct": 5,
|
|
"n_planned": 15,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 15,
|
|
"n_valid": 15,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 1.102096986624212,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"tool_selection": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.00027882058159227585,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9859023175222424,
|
|
"n": 12
|
|
}
|
|
],
|
|
"ece": 0.014097682477757556,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.052782755985390395,
|
|
"p95_s": 0.05833391500636935
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"tradeoff": {
|
|
"accuracy": 0.6666666666666666,
|
|
"brier_mean": 0.3172021465853196,
|
|
"calibration_n": 6,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6392348025938888,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7449709949864798,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8562976654414707,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9682766780586276,
|
|
"n": 1
|
|
}
|
|
],
|
|
"ece": 0.30877110020995474,
|
|
"n": 6
|
|
},
|
|
"latency": {
|
|
"n": 6,
|
|
"p50_s": 0.06587484749616124,
|
|
"p95_s": 0.07062355398375075
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 6,
|
|
"n_correct": 4,
|
|
"n_planned": 6,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 6,
|
|
"n_valid": 6,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"trap": {
|
|
"accuracy": 0.875,
|
|
"brier_mean": 0.0928845979050508,
|
|
"calibration_n": 8,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4660607766073019,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8527907640733415,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9658182629839495,
|
|
"n": 6
|
|
}
|
|
],
|
|
"ece": 0.10229505432878294,
|
|
"n": 8
|
|
},
|
|
"latency": {
|
|
"n": 8,
|
|
"p50_s": 0.05460315546952188,
|
|
"p95_s": 0.055981464550131935
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 8,
|
|
"n_correct": 7,
|
|
"n_planned": 8,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 8,
|
|
"n_valid": 8,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.8643000081318148,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
}
|
|
},
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"probability_sources": [
|
|
"native"
|
|
],
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0,
|
|
"splits": {
|
|
"public": {
|
|
"accuracy": 0.8744588744588745,
|
|
"brier_mean": 0.17806609620099598,
|
|
"calibration_n": 231,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.4,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.3623595989308779,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.46124870941679735,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5637253221582417,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.65,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.648800445448587,
|
|
"n": 20
|
|
},
|
|
{
|
|
"accuracy": 0.7391304347826086,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7523100109863671,
|
|
"n": 23
|
|
},
|
|
{
|
|
"accuracy": 0.9117647058823529,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8641129171311692,
|
|
"n": 34
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9627515382033693,
|
|
"n": 129
|
|
}
|
|
],
|
|
"ece": 0.051072415884039386,
|
|
"n": 231
|
|
},
|
|
"latency": {
|
|
"n": 231,
|
|
"p50_s": 0.054981209977995604,
|
|
"p95_s": 0.14865703746909276
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 231,
|
|
"n_correct": 202,
|
|
"n_planned": 231,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 231,
|
|
"n_valid": 231,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.26234367203481096,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
}
|
|
},
|
|
"version": "jevbench-v1"
|
|
}
|