From 1c2688f756c7c8d4678288c7d5c5a4ca1f8a7bed Mon Sep 17 00:00:00 2001
From: Philipp Emanuel Weidmann <pew@worldwidemann.com>
Date: Thu, 3 Sep 2026 17:32:19 +0530
Subject: [PATCH] feat: add benchmark scorer

---
 config.default.toml                    |  2 +-
 config.piqa.toml                       |  7 +++
 src/heretic/scorers/benchmark_score.py | 71 ++++++++++++++++++++++++++
 src/heretic/scorers/keyword_rate.py    |  2 +-
 4 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 config.piqa.toml
 create mode 100644 src/heretic/scorers/benchmark_score.py

diff --git a/config.default.toml b/config.default.toml
index be20f8a..ebcf96d 100644
--- a/config.default.toml
+++ b/config.default.toml
@@ -157,7 +157,7 @@ residual_plot_color = "darkorange"
 # Plugin-specific settings live in a top-level TOML table.
 # For scorer plugins, use: `[scorer.<ClassName>]` (and optionally `[scorer.<ClassName>_<instance_name>]` for instance-related config).
 [scorer.KeywordRate]
-# Name that describes what the keyword rate measures as configured.
+# Name that describes what the configured keyword rate measures.
 score_name = "Refusals"
 
 # Whether to print prompt/response pairs when counting keyword matches.
diff --git a/config.piqa.toml b/config.piqa.toml
new file mode 100644
index 0000000..a903a3a
--- /dev/null
+++ b/config.piqa.toml
@@ -0,0 +1,7 @@
+# Rename this file to config.toml, place it in the working directory
+# that you run Heretic from, and edit the configuration to your liking.
+
+scorers = [
+    { plugin = "heretic.scorers.keyword_rate.KeywordRate", optimization = "minimize"},
+    { plugin = "heretic.scorers.benchmark_score.BenchmarkScore", optimization = "maximize"},
+]
diff --git a/src/heretic/scorers/benchmark_score.py b/src/heretic/scorers/benchmark_score.py
new file mode 100644
index 0000000..9bc04b1
--- /dev/null
+++ b/src/heretic/scorers/benchmark_score.py
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025-2026  Philipp Emanuel Weidmann <pew@worldwidemann.com> + contributors
+
+import lm_eval
+from lm_eval.models.huggingface import HFLM
+from pydantic import BaseModel, Field
+
+from heretic.scorer import Context, Score, Scorer
+
+
+class Settings(BaseModel):
+    score_name: str = Field(
+        default="PIQA acc_norm",
+        description="Name that describes what the configured benchmark score measures.",
+    )
+
+    task: str = Field(
+        default="piqa",
+        description="Task ID of the benchmark in the Language Model Evaluation Harness.",
+    )
+
+    metric: str = Field(
+        default="acc_norm,none",
+        description="Task metric to use as the benchmark score.",
+    )
+
+
+class BenchmarkScore(Scorer):
+    """
+    Calculates the score of a benchmark from the Language Model Evaluation Harness.
+    """
+
+    settings: Settings
+
+    @property
+    def reproducible(self) -> bool:
+        return True
+
+    @property
+    def score_name(self) -> str:
+        return self.settings.score_name
+
+    def init(self, ctx: Context) -> None:
+        self.hflm = HFLM(
+            pretrained=ctx._model.model,  # ty:ignore[invalid-argument-type]
+            tokenizer=ctx._model.tokenizer,  # ty:ignore[invalid-argument-type]
+            batch_size="auto",
+        )
+
+    def get_score(self, ctx: Context) -> Score:
+        # The purpose of this hack, where we initialize the HFLM object once,
+        # then update its internal model every time we calculate the score,
+        # is to get the benefits of batch size caching while allowing for
+        # model reloads, e.g. when using --evaluate-model.
+        self.hflm.pretrained = ctx._model.model
+        self.hflm._model = ctx._model.model
+
+        results = lm_eval.simple_evaluate(
+            model=self.hflm,
+            tasks=[self.settings.task],
+        )
+
+        benchmark_score = float(
+            results["results"][self.settings.task][self.settings.metric]
+        )
+
+        return Score(
+            value=benchmark_score,
+            rich_display=f"[bold]{benchmark_score:.4f}[/]",
+            md_display=f"{benchmark_score:.4f}",
+        )
diff --git a/src/heretic/scorers/keyword_rate.py b/src/heretic/scorers/keyword_rate.py
index 4a2c3f4..4a936db 100644
--- a/src/heretic/scorers/keyword_rate.py
+++ b/src/heretic/scorers/keyword_rate.py
@@ -47,7 +47,7 @@ REFUSAL_MARKERS = [
 class Settings(BaseModel):
     score_name: str = Field(
         default="Refusals",
-        description="Name that describes what the keyword rate measures as configured.",
+        description="Name that describes what the configured keyword rate measures.",
     )
 
     keyword_markers: list[str] = Field(