{ "accuracy": 0.8744588744588745, "brier_mean": 0.17806609620099598, "calibration_n": 231, "complete": true, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.4, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3623595989308779, "n": 10 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.46124870941679735, "n": 5 }, { "accuracy": 0.8, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5637253221582417, "n": 10 }, { "accuracy": 0.65, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.648800445448587, "n": 20 }, { "accuracy": 0.7391304347826086, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7523100109863671, "n": 23 }, { "accuracy": 0.9117647058823529, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8641129171311692, "n": 34 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9627515382033693, "n": 129 } ], "ece": 0.051072415884039386, "n": 231 }, "latency": { "n": 231, "p50_s": 0.05428842600667849, "p95_s": 0.14888899750076234 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": 0.0, "macro_accuracy": 0.8725744098809114, "model_identities": [ "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd" ], "n_attempted": 231, "n_correct": 202, "n_planned": 231, "n_renormalized": 0, "n_scorable": 231, "n_valid": 231, "operational_success": 1.0, "ordinal_mae": 0.26234367203481096, "paraphrase_consistency": { "agree": 35, "agreement": 0.9722222222222222, "both_correct_rate_all_pairs": 0.9722222222222222, "both_valid": 36, "pairs": 36 }, "per_family": { "adequacy": { "accuracy": 1.0, "brier_mean": 0.04603689738961828, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5930110454411601, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7557061813028185, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8878357373215198, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9520119763886462, "n": 8 } ], "ece": 0.104959623958651, "n": 12 }, "latency": { "n": 12, "p50_s": 0.052879591967212036, "p95_s": 0.054876283335033804 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "adversarial": { "accuracy": 1.0, "brier_mean": 0.011617825043304442, "calibration_n": 6, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8627679157764538, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9629914990167525, "n": 4 } ], "ece": 0.07041636206334703, "n": 6 }, "latency": { "n": 6, "p50_s": 0.05504016700433567, "p95_s": 0.05621324274397921 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 6, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": 0.10376329334202228, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ambiguous": { "accuracy": 0.7142857142857143, "brier_mean": 0.3223035707740803, "calibration_n": 7, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.3333333333333333, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6479833643674017, "n": 3 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9463672181616865, "n": 4 } ], "ece": 0.16549731720792277, "n": 7 }, "latency": { "n": 7, "p50_s": 0.06194691703421995, "p95_s": 0.0629515228967648 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 7, "n_correct": 5, "n_planned": 7, "n_renormalized": 0, "n_scorable": 7, "n_valid": 7, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "brier_mean": 0.0019479746093977082, "calibration_n": 24, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9683153887441366, "n": 24 } ], "ece": 0.031684611255863415, "n": 24 }, "latency": { "n": 24, "p50_s": 0.053058729477925226, "p95_s": 0.0536909365124302 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 24, "n_correct": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_valid": 24, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "fact": { "accuracy": 1.0, "brier_mean": 0.00253357674402684, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9707540110368739, "n": 12 } ], "ece": 0.029245988963126135, "n": 12 }, "latency": { "n": 12, "p50_s": 0.05440999200800434, "p95_s": 0.0573352141684154 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 0.9583333333333334, "brier_mean": 0.04236379090188356, "calibration_n": 24, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4791247461286767, "n": 1 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.697318944283325, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7504260155024499, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8623501745620707, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.97124439087109, "n": 17 } ], "ece": 0.0909478584310712, "n": 24 }, "latency": { "n": 24, "p50_s": 0.053307871479773894, "p95_s": 0.060923036502208545 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 24, "n_correct": 23, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_valid": 24, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "judge_hard": { "accuracy": 0.9411764705882353, "brier_mean": 0.1769820408934362, "calibration_n": 17, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5469149563843179, "n": 1 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6659927647326704, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7429117727934124, "n": 5 }, { "accuracy": 0.8, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8717155645981629, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9748315019610428, "n": 5 } ], "ece": 0.15040904282950293, "n": 17 }, "latency": { "n": 17, "p50_s": 0.054366590979043394, "p95_s": 13.006528226204635 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 17, "n_correct": 16, "n_planned": 17, "n_renormalized": 0, "n_scorable": 17, "n_valid": 17, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "long_policy": { "accuracy": 0.6842105263157895, "brier_mean": 0.488183253400819, "calibration_n": 19, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.32987837012316157, "n": 2 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4247953847281636, "n": 1 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5930110454411601, "n": 1 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6374422769566587, "n": 3 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7434209921956753, "n": 4 }, { "accuracy": 0.8333333333333334, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8591073374831182, "n": 6 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9264976568409564, "n": 2 } ], "ece": 0.13342281306158954, "n": 19 }, "latency": { "n": 19, "p50_s": 0.14695013902382925, "p95_s": 0.18794695567921732 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 19, "n_correct": 13, "n_planned": 19, "n_renormalized": 0, "n_scorable": 19, "n_valid": 19, "operational_success": 1.0, "ordinal_mae": 0.8438052410179581, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "multi_hop": { "accuracy": 0.8333333333333334, "brier_mean": 0.2821094188229273, "calibration_n": 18, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3946907024781775, "n": 1 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.75, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5453134547809111, "n": 4 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6537803635138819, "n": 2 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7674221407596936, "n": 4 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8512239723447685, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9325685206539565, "n": 2 } ], "ece": 0.22582924507652405, "n": 18 }, "latency": { "n": 18, "p50_s": 0.14008955997996964, "p95_s": 0.15341312710661442 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 18, "n_correct": 15, "n_planned": 18, "n_renormalized": 0, "n_scorable": 18, "n_valid": 18, "operational_success": 1.0, "ordinal_mae": 0.3268150682115487, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "brier_mean": 0.006310346320679678, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8500853530913438, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.956899632823367, "n": 11 } ], "ece": 0.05200155715430161, "n": 12 }, "latency": { "n": 12, "p50_s": 0.053522854985203594, "p95_s": 0.05490196594328153 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": 0.0531333548567569, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 1.0, "brier_mean": 0.020422084182937634, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7557061813028185, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8742406957013814, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9513615324190499, "n": 9 } ], "ece": 0.07779655296024748, "n": 12 }, "latency": { "n": 12, "p50_s": 0.053544344496913254, "p95_s": 0.05435312601330225 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "probability": { "accuracy": 0.7, "brier_mean": 0.46568152828081544, "calibration_n": 10, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5777854893190272, "n": 1 }, { "accuracy": 0.8, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6325445353204342, "n": 5 }, { "accuracy": 0.3333333333333333, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7561351202525173, "n": 3 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8908360355378411, "n": 1 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.26370611592985127, "n": 10 }, "latency": { "n": 10, "p50_s": 0.06932540450361557, "p95_s": 0.0942387860675808 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 10, "n_correct": 7, "n_planned": 10, "n_renormalized": 0, "n_scorable": 10, "n_valid": 10, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 1.0, "brier_mean": 0.041188901489954015, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6801954980304856, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7877520218832728, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8825011701644436, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9364805526689252, "n": 7 } ], "ece": 0.12762423108703272, "n": 12 }, "latency": { "n": 12, "p50_s": 0.052542628545779735, "p95_s": 0.05301162832183763 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing_hard": { "accuracy": 1.0, "brier_mean": 0.0051631143261297256, "calibration_n": 5, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9423656177532307, "n": 5 } ], "ece": 0.05763438224676931, "n": 5 }, "latency": { "n": 5, "p50_s": 0.054628726036753505, "p95_s": 0.05692764220293611 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 5, "n_correct": 5, "n_planned": 5, "n_renormalized": 0, "n_scorable": 5, "n_valid": 5, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "temporal_numeric": { "accuracy": 0.3333333333333333, "brier_mean": 0.7767759878778288, "calibration_n": 15, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.2857142857142857, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36702122094061124, "n": 7 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4681313198099223, "n": 2 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5726384329365539, "n": 2 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6472762711048763, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7318060856600669, "n": 1 }, { "accuracy": 0.5, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8512242138235586, "n": 2 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.2179071110112772, "n": 15 }, "latency": { "n": 15, "p50_s": 0.06334954197518528, "p95_s": 0.08096886847633866 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 15, "n_correct": 5, "n_planned": 15, "n_renormalized": 0, "n_scorable": 15, "n_valid": 15, "operational_success": 1.0, "ordinal_mae": 1.102096986624212, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tool_selection": { "accuracy": 1.0, "brier_mean": 0.00027882058159227585, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9859023175222424, "n": 12 } ], "ece": 0.014097682477757556, "n": 12 }, "latency": { "n": 12, "p50_s": 0.05312584349303506, "p95_s": 0.05412412802688778 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tradeoff": { "accuracy": 0.6666666666666666, "brier_mean": 0.3172021465853196, "calibration_n": 6, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6392348025938888, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7449709949864798, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8562976654414707, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9682766780586276, "n": 1 } ], "ece": 0.30877110020995474, "n": 6 }, "latency": { "n": 6, "p50_s": 0.06751805800013244, "p95_s": 0.06984318800095934 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 4, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "trap": { "accuracy": 0.875, "brier_mean": 0.0928845979050508, "calibration_n": 8, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4660607766073019, "n": 1 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8527907640733415, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9658182629839495, "n": 6 } ], "ece": 0.10229505432878294, "n": 8 }, "latency": { "n": 8, "p50_s": 0.055600558494916186, "p95_s": 0.05691805843380279 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 8, "n_correct": 7, "n_planned": 8, "n_renormalized": 0, "n_scorable": 8, "n_valid": 8, "operational_success": 1.0, "ordinal_mae": 0.8643000081318148, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": 0.0, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.8744588744588745, "brier_mean": 0.17806609620099598, "calibration_n": 231, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.4, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3623595989308779, "n": 10 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.46124870941679735, "n": 5 }, { "accuracy": 0.8, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5637253221582417, "n": 10 }, { "accuracy": 0.65, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.648800445448587, "n": 20 }, { "accuracy": 0.7391304347826086, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7523100109863671, "n": 23 }, { "accuracy": 0.9117647058823529, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8641129171311692, "n": 34 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9627515382033693, "n": 129 } ], "ece": 0.051072415884039386, "n": 231 }, "latency": { "n": 231, "p50_s": 0.05428842600667849, "p95_s": 0.14888899750076234 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 231, "n_correct": 202, "n_planned": 231, "n_renormalized": 0, "n_scorable": 231, "n_valid": 231, "operational_success": 1.0, "ordinal_mae": 0.26234367203481096, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }