vh
f65e27b08f
The ~6.5-9 s first-call-per-bucket autotune lived in the container's writable layer and died on every recreate. 0.1.3 creates /tmp/triton-cache in the image owned by 10001 so the named volume intern-decision_triton-cache inherits a writable mount point, and compose mounts it. Bucket model PROVEN, not inferred: 2,048-token buckets, 16 up to 32,768. After one warmed call per bucket, 12 random sizes across 8k-32k were all warm (worst 2.09 s); cold entries cost 6.5-9 s. Full cold warm-up 109 s; warm re-run 17 s. scripts/intern-decision-warmup: one noul call per bucket, MAX_TOKENS from /health, two-point live calibration of the tokenizer's linear token model (a single probe overcorrects and the aim oscillates around the bucket edge), per-bucket wall times, non-zero exit on a missed bucket. Run it after an IMAGE CHANGE only; the volume carries ordinary recreates (measured: force-recreate, then a warmed 32k call answered in 2.11 s). Acceptance on 0.1.3: JevBench 202/231, hard 83/111, 0 diffs / 924; warm 32k GPU 1 peak 15,218 MiB (budget 15,220; a COLD autotune touched 15,224 once, README caveat); /decide answers. Artifacts in the acceptance dir.
2142 lines
49 KiB
JSON
2142 lines
49 KiB
JSON
{
|
|
"accuracy": 0.8744588744588745,
|
|
"brier_mean": 0.17806609620099598,
|
|
"calibration_n": 231,
|
|
"complete": true,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.4,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.3623595989308779,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.46124870941679735,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5637253221582417,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.65,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.648800445448587,
|
|
"n": 20
|
|
},
|
|
{
|
|
"accuracy": 0.7391304347826086,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7523100109863671,
|
|
"n": 23
|
|
},
|
|
{
|
|
"accuracy": 0.9117647058823529,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8641129171311692,
|
|
"n": 34
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9627515382033693,
|
|
"n": 129
|
|
}
|
|
],
|
|
"ece": 0.051072415884039386,
|
|
"n": 231
|
|
},
|
|
"latency": {
|
|
"n": 231,
|
|
"p50_s": 0.05428842600667849,
|
|
"p95_s": 0.14888899750076234
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"ledger_charged_usd": 0.0,
|
|
"macro_accuracy": 0.8725744098809114,
|
|
"model_identities": [
|
|
"Intern-Decision-4B@0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd"
|
|
],
|
|
"n_attempted": 231,
|
|
"n_correct": 202,
|
|
"n_planned": 231,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 231,
|
|
"n_valid": 231,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.26234367203481096,
|
|
"paraphrase_consistency": {
|
|
"agree": 35,
|
|
"agreement": 0.9722222222222222,
|
|
"both_correct_rate_all_pairs": 0.9722222222222222,
|
|
"both_valid": 36,
|
|
"pairs": 36
|
|
},
|
|
"per_family": {
|
|
"adequacy": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.04603689738961828,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5930110454411601,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7557061813028185,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8878357373215198,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9520119763886462,
|
|
"n": 8
|
|
}
|
|
],
|
|
"ece": 0.104959623958651,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.052879591967212036,
|
|
"p95_s": 0.054876283335033804
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"adversarial": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.011617825043304442,
|
|
"calibration_n": 6,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8627679157764538,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9629914990167525,
|
|
"n": 4
|
|
}
|
|
],
|
|
"ece": 0.07041636206334703,
|
|
"n": 6
|
|
},
|
|
"latency": {
|
|
"n": 6,
|
|
"p50_s": 0.05504016700433567,
|
|
"p95_s": 0.05621324274397921
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 6,
|
|
"n_correct": 6,
|
|
"n_planned": 6,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 6,
|
|
"n_valid": 6,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.10376329334202228,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"ambiguous": {
|
|
"accuracy": 0.7142857142857143,
|
|
"brier_mean": 0.3223035707740803,
|
|
"calibration_n": 7,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.3333333333333333,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6479833643674017,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9463672181616865,
|
|
"n": 4
|
|
}
|
|
],
|
|
"ece": 0.16549731720792277,
|
|
"n": 7
|
|
},
|
|
"latency": {
|
|
"n": 7,
|
|
"p50_s": 0.06194691703421995,
|
|
"p95_s": 0.0629515228967648
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 7,
|
|
"n_correct": 5,
|
|
"n_planned": 7,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 7,
|
|
"n_valid": 7,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"extraction": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.0019479746093977082,
|
|
"calibration_n": 24,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9683153887441366,
|
|
"n": 24
|
|
}
|
|
],
|
|
"ece": 0.031684611255863415,
|
|
"n": 24
|
|
},
|
|
"latency": {
|
|
"n": 24,
|
|
"p50_s": 0.053058729477925226,
|
|
"p95_s": 0.0536909365124302
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 24,
|
|
"n_correct": 24,
|
|
"n_planned": 24,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 24,
|
|
"n_valid": 24,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"fact": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.00253357674402684,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9707540110368739,
|
|
"n": 12
|
|
}
|
|
],
|
|
"ece": 0.029245988963126135,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.05440999200800434,
|
|
"p95_s": 0.0573352141684154
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"intent": {
|
|
"accuracy": 0.9583333333333334,
|
|
"brier_mean": 0.04236379090188356,
|
|
"calibration_n": 24,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4791247461286767,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.697318944283325,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7504260155024499,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8623501745620707,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.97124439087109,
|
|
"n": 17
|
|
}
|
|
],
|
|
"ece": 0.0909478584310712,
|
|
"n": 24
|
|
},
|
|
"latency": {
|
|
"n": 24,
|
|
"p50_s": 0.053307871479773894,
|
|
"p95_s": 0.060923036502208545
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 24,
|
|
"n_correct": 23,
|
|
"n_planned": 24,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 24,
|
|
"n_valid": 24,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"judge_hard": {
|
|
"accuracy": 0.9411764705882353,
|
|
"brier_mean": 0.1769820408934362,
|
|
"calibration_n": 17,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5469149563843179,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6659927647326704,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7429117727934124,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8717155645981629,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9748315019610428,
|
|
"n": 5
|
|
}
|
|
],
|
|
"ece": 0.15040904282950293,
|
|
"n": 17
|
|
},
|
|
"latency": {
|
|
"n": 17,
|
|
"p50_s": 0.054366590979043394,
|
|
"p95_s": 13.006528226204635
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 17,
|
|
"n_correct": 16,
|
|
"n_planned": 17,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 17,
|
|
"n_valid": 17,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"long_policy": {
|
|
"accuracy": 0.6842105263157895,
|
|
"brier_mean": 0.488183253400819,
|
|
"calibration_n": 19,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.32987837012316157,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4247953847281636,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5930110454411601,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 0.6666666666666666,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6374422769566587,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7434209921956753,
|
|
"n": 4
|
|
},
|
|
{
|
|
"accuracy": 0.8333333333333334,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8591073374831182,
|
|
"n": 6
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9264976568409564,
|
|
"n": 2
|
|
}
|
|
],
|
|
"ece": 0.13342281306158954,
|
|
"n": 19
|
|
},
|
|
"latency": {
|
|
"n": 19,
|
|
"p50_s": 0.14695013902382925,
|
|
"p95_s": 0.18794695567921732
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 19,
|
|
"n_correct": 13,
|
|
"n_planned": 19,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 19,
|
|
"n_valid": 19,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.8438052410179581,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"multi_hop": {
|
|
"accuracy": 0.8333333333333334,
|
|
"brier_mean": 0.2821094188229273,
|
|
"calibration_n": 18,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.3946907024781775,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.75,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5453134547809111,
|
|
"n": 4
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6537803635138819,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7674221407596936,
|
|
"n": 4
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8512239723447685,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9325685206539565,
|
|
"n": 2
|
|
}
|
|
],
|
|
"ece": 0.22582924507652405,
|
|
"n": 18
|
|
},
|
|
"latency": {
|
|
"n": 18,
|
|
"p50_s": 0.14008955997996964,
|
|
"p95_s": 0.15341312710661442
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 18,
|
|
"n_correct": 15,
|
|
"n_planned": 18,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 18,
|
|
"n_valid": 18,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.3268150682115487,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"ordinal": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.006310346320679678,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8500853530913438,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.956899632823367,
|
|
"n": 11
|
|
}
|
|
],
|
|
"ece": 0.05200155715430161,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.053522854985203594,
|
|
"p95_s": 0.05490196594328153
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.0531333548567569,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"policy": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.020422084182937634,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7557061813028185,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8742406957013814,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9513615324190499,
|
|
"n": 9
|
|
}
|
|
],
|
|
"ece": 0.07779655296024748,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.053544344496913254,
|
|
"p95_s": 0.05435312601330225
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"probability": {
|
|
"accuracy": 0.7,
|
|
"brier_mean": 0.46568152828081544,
|
|
"calibration_n": 10,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5777854893190272,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6325445353204342,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.3333333333333333,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7561351202525173,
|
|
"n": 3
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8908360355378411,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
}
|
|
],
|
|
"ece": 0.26370611592985127,
|
|
"n": 10
|
|
},
|
|
"latency": {
|
|
"n": 10,
|
|
"p50_s": 0.06932540450361557,
|
|
"p95_s": 0.0942387860675808
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 10,
|
|
"n_correct": 7,
|
|
"n_planned": 10,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 10,
|
|
"n_valid": 10,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"routing": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.041188901489954015,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6801954980304856,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7877520218832728,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8825011701644436,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9364805526689252,
|
|
"n": 7
|
|
}
|
|
],
|
|
"ece": 0.12762423108703272,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.052542628545779735,
|
|
"p95_s": 0.05301162832183763
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"routing_hard": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.0051631143261297256,
|
|
"calibration_n": 5,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9423656177532307,
|
|
"n": 5
|
|
}
|
|
],
|
|
"ece": 0.05763438224676931,
|
|
"n": 5
|
|
},
|
|
"latency": {
|
|
"n": 5,
|
|
"p50_s": 0.054628726036753505,
|
|
"p95_s": 0.05692764220293611
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 5,
|
|
"n_correct": 5,
|
|
"n_planned": 5,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 5,
|
|
"n_valid": 5,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"temporal_numeric": {
|
|
"accuracy": 0.3333333333333333,
|
|
"brier_mean": 0.7767759878778288,
|
|
"calibration_n": 15,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.2857142857142857,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.36702122094061124,
|
|
"n": 7
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4681313198099223,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5726384329365539,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6472762711048763,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7318060856600669,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 0.5,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8512242138235586,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
}
|
|
],
|
|
"ece": 0.2179071110112772,
|
|
"n": 15
|
|
},
|
|
"latency": {
|
|
"n": 15,
|
|
"p50_s": 0.06334954197518528,
|
|
"p95_s": 0.08096886847633866
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 15,
|
|
"n_correct": 5,
|
|
"n_planned": 15,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 15,
|
|
"n_valid": 15,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 1.102096986624212,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"tool_selection": {
|
|
"accuracy": 1.0,
|
|
"brier_mean": 0.00027882058159227585,
|
|
"calibration_n": 12,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9859023175222424,
|
|
"n": 12
|
|
}
|
|
],
|
|
"ece": 0.014097682477757556,
|
|
"n": 12
|
|
},
|
|
"latency": {
|
|
"n": 12,
|
|
"p50_s": 0.05312584349303506,
|
|
"p95_s": 0.05412412802688778
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 12,
|
|
"n_correct": 12,
|
|
"n_planned": 12,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 12,
|
|
"n_valid": 12,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"tradeoff": {
|
|
"accuracy": 0.6666666666666666,
|
|
"brier_mean": 0.3172021465853196,
|
|
"calibration_n": 6,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.6392348025938888,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7449709949864798,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8562976654414707,
|
|
"n": 2
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9682766780586276,
|
|
"n": 1
|
|
}
|
|
],
|
|
"ece": 0.30877110020995474,
|
|
"n": 6
|
|
},
|
|
"latency": {
|
|
"n": 6,
|
|
"p50_s": 0.06751805800013244,
|
|
"p95_s": 0.06984318800095934
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 6,
|
|
"n_correct": 4,
|
|
"n_planned": 6,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 6,
|
|
"n_valid": 6,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": null,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
},
|
|
"trap": {
|
|
"accuracy": 0.875,
|
|
"brier_mean": 0.0928845979050508,
|
|
"calibration_n": 8,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.4660607766073019,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8527907640733415,
|
|
"n": 1
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9658182629839495,
|
|
"n": 6
|
|
}
|
|
],
|
|
"ece": 0.10229505432878294,
|
|
"n": 8
|
|
},
|
|
"latency": {
|
|
"n": 8,
|
|
"p50_s": 0.055600558494916186,
|
|
"p95_s": 0.05691805843380279
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 8,
|
|
"n_correct": 7,
|
|
"n_planned": 8,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 8,
|
|
"n_valid": 8,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.8643000081318148,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
}
|
|
},
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"probability_sources": [
|
|
"native"
|
|
],
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0,
|
|
"splits": {
|
|
"public": {
|
|
"accuracy": 0.8744588744588745,
|
|
"brier_mean": 0.17806609620099598,
|
|
"calibration_n": 231,
|
|
"cost_basis": [
|
|
"zero_route_fee_compute_excluded"
|
|
],
|
|
"coverage": 1.0,
|
|
"ece": {
|
|
"bins": [
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.1,
|
|
"lo": 0.0,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.2,
|
|
"lo": 0.1,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": null,
|
|
"hi": 0.3,
|
|
"lo": 0.2,
|
|
"mean_confidence": null,
|
|
"n": 0
|
|
},
|
|
{
|
|
"accuracy": 0.4,
|
|
"hi": 0.4,
|
|
"lo": 0.3,
|
|
"mean_confidence": 0.3623595989308779,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.0,
|
|
"hi": 0.5,
|
|
"lo": 0.4,
|
|
"mean_confidence": 0.46124870941679735,
|
|
"n": 5
|
|
},
|
|
{
|
|
"accuracy": 0.8,
|
|
"hi": 0.6,
|
|
"lo": 0.5,
|
|
"mean_confidence": 0.5637253221582417,
|
|
"n": 10
|
|
},
|
|
{
|
|
"accuracy": 0.65,
|
|
"hi": 0.7,
|
|
"lo": 0.6,
|
|
"mean_confidence": 0.648800445448587,
|
|
"n": 20
|
|
},
|
|
{
|
|
"accuracy": 0.7391304347826086,
|
|
"hi": 0.8,
|
|
"lo": 0.7,
|
|
"mean_confidence": 0.7523100109863671,
|
|
"n": 23
|
|
},
|
|
{
|
|
"accuracy": 0.9117647058823529,
|
|
"hi": 0.9,
|
|
"lo": 0.8,
|
|
"mean_confidence": 0.8641129171311692,
|
|
"n": 34
|
|
},
|
|
{
|
|
"accuracy": 1.0,
|
|
"hi": 1.0,
|
|
"lo": 0.9,
|
|
"mean_confidence": 0.9627515382033693,
|
|
"n": 129
|
|
}
|
|
],
|
|
"ece": 0.051072415884039386,
|
|
"n": 231
|
|
},
|
|
"latency": {
|
|
"n": 231,
|
|
"p50_s": 0.05428842600667849,
|
|
"p95_s": 0.14888899750076234
|
|
},
|
|
"latency_failures": {
|
|
"n": 0,
|
|
"p50_s": null,
|
|
"p95_s": null
|
|
},
|
|
"n_attempted": 231,
|
|
"n_correct": 202,
|
|
"n_planned": 231,
|
|
"n_renormalized": 0,
|
|
"n_scorable": 231,
|
|
"n_valid": 231,
|
|
"operational_success": 1.0,
|
|
"ordinal_mae": 0.26234367203481096,
|
|
"price_per_1000_decisions_usd": 0.0,
|
|
"schema_validity": 1.0,
|
|
"schema_validity_strict": 1.0
|
|
}
|
|
},
|
|
"version": "jevbench-v1"
|
|
}
|