feat: add benchmark scorer (#444)

* fix: improve print output of scorers * feat: add benchmark scorer
2026-09-25 13:31:33 -07:00 · 2026-09-03 17:49:46 +05:30
parent c7a44f0db7
commit 95dda4c4db
11 changed files with 113 additions and 40 deletions
@@ -157,6 +157,9 @@ residual_plot_color = "darkorange"
 # Plugin-specific settings live in a top-level TOML table.
 # For scorer plugins, use: `[scorer.<ClassName>]` (and optionally `[scorer.<ClassName>_<instance_name>]` for instance-related config).
 [scorer.KeywordRate]
+# Name that describes what the configured keyword rate measures.
+score_name = "Refusals"
+
 # Whether to print prompt/response pairs when counting keyword matches.
 print_responses = false