{ "accuracy": 0.8744588744588745, "brier_mean": 0.17806609620099598, "calibration_n": 231, "complete": true, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.4, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3623595989308779, "n": 10 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.46124870941679735, "n": 5 }, { "accuracy": 0.8, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5637253221582417, "n": 10 }, { "accuracy": 0.65, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.648800445448587, "n": 20 }, { "accuracy": 0.7391304347826086, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7523100109863671, "n": 23 }, { "accuracy": 0.9117647058823529, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8641129171311692, "n": 34 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9627515382033693, "n": 129 } ], "ece": 0.051072415884039386, "n": 231 }, "latency": { "n": 231, "p50_s": 0.054981209977995604, "p95_s": 0.14865703746909276 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": 0.0, "macro_accuracy": 0.8725744098809114, "model_identities": [ "Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd" ], "n_attempted": 231, "n_correct": 202, "n_planned": 231, "n_renormalized": 0, "n_scorable": 231, "n_valid": 231, "operational_success": 1.0, "ordinal_mae": 0.26234367203481096, "paraphrase_consistency": { "agree": 35, "agreement": 0.9722222222222222, "both_correct_rate_all_pairs": 0.9722222222222222, "both_valid": 36, "pairs": 36 }, "per_family": { "adequacy": { "accuracy": 1.0, "brier_mean": 0.04603689738961828, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5930110454411601, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7557061813028185, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8878357373215198, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9520119763886462, "n": 8 } ], "ece": 0.104959623958651, "n": 12 }, "latency": { "n": 12, "p50_s": 0.05385151947848499, "p95_s": 0.05537226824963 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "adversarial": { "accuracy": 1.0, "brier_mean": 0.011617825043304442, "calibration_n": 6, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8627679157764538, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9629914990167525, "n": 4 } ], "ece": 0.07041636206334703, "n": 6 }, "latency": { "n": 6, "p50_s": 0.05564080100157298, "p95_s": 0.058122831251239404 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 6, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": 0.10376329334202228, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ambiguous": { "accuracy": 0.7142857142857143, "brier_mean": 0.3223035707740803, "calibration_n": 7, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.3333333333333333, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6479833643674017, "n": 3 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9463672181616865, "n": 4 } ], "ece": 0.16549731720792277, "n": 7 }, "latency": { "n": 7, "p50_s": 0.06240525998873636, "p95_s": 0.06425336602260359 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 7, "n_correct": 5, "n_planned": 7, "n_renormalized": 0, "n_scorable": 7, "n_valid": 7, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "brier_mean": 0.0019479746093977082, "calibration_n": 24, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9683153887441366, "n": 24 } ], "ece": 0.031684611255863415, "n": 24 }, "latency": { "n": 24, "p50_s": 0.05352575849974528, "p95_s": 0.05614419543999247 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 24, "n_correct": 24, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_valid": 24, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "fact": { "accuracy": 1.0, "brier_mean": 0.00253357674402684, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9707540110368739, "n": 12 } ], "ece": 0.029245988963126135, "n": 12 }, "latency": { "n": 12, "p50_s": 0.052916569489752874, "p95_s": 0.053868782028439456 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 0.9583333333333334, "brier_mean": 0.04236379090188356, "calibration_n": 24, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4791247461286767, "n": 1 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.697318944283325, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7504260155024499, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8623501745620707, "n": 3 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.97124439087109, "n": 17 } ], "ece": 0.0909478584310712, "n": 24 }, "latency": { "n": 24, "p50_s": 0.05407417201786302, "p95_s": 0.059262538660550486 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 24, "n_correct": 23, "n_planned": 24, "n_renormalized": 0, "n_scorable": 24, "n_valid": 24, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "judge_hard": { "accuracy": 0.9411764705882353, "brier_mean": 0.1769820408934362, "calibration_n": 17, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5469149563843179, "n": 1 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6659927647326704, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7429117727934124, "n": 5 }, { "accuracy": 0.8, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8717155645981629, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9748315019610428, "n": 5 } ], "ece": 0.15040904282950293, "n": 17 }, "latency": { "n": 17, "p50_s": 0.054450018040370196, "p95_s": 0.055332889617420736 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 17, "n_correct": 16, "n_planned": 17, "n_renormalized": 0, "n_scorable": 17, "n_valid": 17, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "long_policy": { "accuracy": 0.6842105263157895, "brier_mean": 0.488183253400819, "calibration_n": 19, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.32987837012316157, "n": 2 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4247953847281636, "n": 1 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5930110454411601, "n": 1 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6374422769566587, "n": 3 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7434209921956753, "n": 4 }, { "accuracy": 0.8333333333333334, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8591073374831182, "n": 6 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9264976568409564, "n": 2 } ], "ece": 0.13342281306158954, "n": 19 }, "latency": { "n": 19, "p50_s": 0.15093205898301676, "p95_s": 0.8460185076109925 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 19, "n_correct": 13, "n_planned": 19, "n_renormalized": 0, "n_scorable": 19, "n_valid": 19, "operational_success": 1.0, "ordinal_mae": 0.8438052410179581, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "multi_hop": { "accuracy": 0.8333333333333334, "brier_mean": 0.2821094188229273, "calibration_n": 18, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3946907024781775, "n": 1 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.75, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5453134547809111, "n": 4 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6537803635138819, "n": 2 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7674221407596936, "n": 4 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8512239723447685, "n": 5 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9325685206539565, "n": 2 } ], "ece": 0.22582924507652405, "n": 18 }, "latency": { "n": 18, "p50_s": 0.14022018198738806, "p95_s": 0.1504995027236873 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 18, "n_correct": 15, "n_planned": 18, "n_renormalized": 0, "n_scorable": 18, "n_valid": 18, "operational_success": 1.0, "ordinal_mae": 0.3268150682115487, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 1.0, "brier_mean": 0.006310346320679678, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8500853530913438, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.956899632823367, "n": 11 } ], "ece": 0.05200155715430161, "n": 12 }, "latency": { "n": 12, "p50_s": 0.054876862006494775, "p95_s": 0.06036159305367618 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": 0.0531333548567569, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 1.0, "brier_mean": 0.020422084182937634, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7557061813028185, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8742406957013814, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9513615324190499, "n": 9 } ], "ece": 0.07779655296024748, "n": 12 }, "latency": { "n": 12, "p50_s": 0.05428065449814312, "p95_s": 0.05681873176072258 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "probability": { "accuracy": 0.7, "brier_mean": 0.46568152828081544, "calibration_n": 10, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5777854893190272, "n": 1 }, { "accuracy": 0.8, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6325445353204342, "n": 5 }, { "accuracy": 0.3333333333333333, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7561351202525173, "n": 3 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8908360355378411, "n": 1 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.26370611592985127, "n": 10 }, "latency": { "n": 10, "p50_s": 0.07672692849882878, "p95_s": 0.09677280441683249 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 10, "n_correct": 7, "n_planned": 10, "n_renormalized": 0, "n_scorable": 10, "n_valid": 10, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 1.0, "brier_mean": 0.041188901489954015, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6801954980304856, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7877520218832728, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8825011701644436, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9364805526689252, "n": 7 } ], "ece": 0.12762423108703272, "n": 12 }, "latency": { "n": 12, "p50_s": 0.05457381851738319, "p95_s": 0.05613349811465014 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing_hard": { "accuracy": 1.0, "brier_mean": 0.0051631143261297256, "calibration_n": 5, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9423656177532307, "n": 5 } ], "ece": 0.05763438224676931, "n": 5 }, "latency": { "n": 5, "p50_s": 0.055770282982848585, "p95_s": 0.05623367396183312 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 5, "n_correct": 5, "n_planned": 5, "n_renormalized": 0, "n_scorable": 5, "n_valid": 5, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "temporal_numeric": { "accuracy": 0.3333333333333333, "brier_mean": 0.7767759878778288, "calibration_n": 15, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.2857142857142857, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.36702122094061124, "n": 7 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4681313198099223, "n": 2 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5726384329365539, "n": 2 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6472762711048763, "n": 1 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7318060856600669, "n": 1 }, { "accuracy": 0.5, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8512242138235586, "n": 2 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.2179071110112772, "n": 15 }, "latency": { "n": 15, "p50_s": 0.06412995798746124, "p95_s": 0.07775642202468588 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 15, "n_correct": 5, "n_planned": 15, "n_renormalized": 0, "n_scorable": 15, "n_valid": 15, "operational_success": 1.0, "ordinal_mae": 1.102096986624212, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tool_selection": { "accuracy": 1.0, "brier_mean": 0.00027882058159227585, "calibration_n": 12, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9859023175222424, "n": 12 } ], "ece": 0.014097682477757556, "n": 12 }, "latency": { "n": 12, "p50_s": 0.052782755985390395, "p95_s": 0.05833391500636935 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tradeoff": { "accuracy": 0.6666666666666666, "brier_mean": 0.3172021465853196, "calibration_n": 6, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6392348025938888, "n": 2 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7449709949864798, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8562976654414707, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9682766780586276, "n": 1 } ], "ece": 0.30877110020995474, "n": 6 }, "latency": { "n": 6, "p50_s": 0.06587484749616124, "p95_s": 0.07062355398375075 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 4, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "trap": { "accuracy": 0.875, "brier_mean": 0.0928845979050508, "calibration_n": 8, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4660607766073019, "n": 1 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8527907640733415, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9658182629839495, "n": 6 } ], "ece": 0.10229505432878294, "n": 8 }, "latency": { "n": 8, "p50_s": 0.05460315546952188, "p95_s": 0.055981464550131935 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 8, "n_correct": 7, "n_planned": 8, "n_renormalized": 0, "n_scorable": 8, "n_valid": 8, "operational_success": 1.0, "ordinal_mae": 0.8643000081318148, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": 0.0, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.8744588744588745, "brier_mean": 0.17806609620099598, "calibration_n": 231, "cost_basis": [ "zero_route_fee_compute_excluded" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.4, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3623595989308779, "n": 10 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.46124870941679735, "n": 5 }, { "accuracy": 0.8, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5637253221582417, "n": 10 }, { "accuracy": 0.65, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.648800445448587, "n": 20 }, { "accuracy": 0.7391304347826086, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7523100109863671, "n": 23 }, { "accuracy": 0.9117647058823529, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8641129171311692, "n": 34 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9627515382033693, "n": 129 } ], "ece": 0.051072415884039386, "n": 231 }, "latency": { "n": 231, "p50_s": 0.054981209977995604, "p95_s": 0.14865703746909276 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 231, "n_correct": 202, "n_planned": 231, "n_renormalized": 0, "n_scorable": 231, "n_valid": 231, "operational_success": 1.0, "ordinal_mae": 0.26234367203481096, "price_per_1000_decisions_usd": 0.0, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }