feat: add reproducible property to plugins (#414)

* feat: add `reproducible` property to plugins

* feat: hard stop reproducibility and actually check `reproducible` field in the evaluator

* chore: flip default

* fix: check if plugin is built-in in repro gate

* fix: ruff

* fix: kld should be reproducible

* feat: move reproducible field to scorer level

* chore: move reproducible field back to plugin level

---------

Co-authored-by: mad-cat-lon <113548315+mad-cat-lon@users.noreply.github.com>

src/heretic/evaluator.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel
 
 from .config import DatasetSpecification, ScorerConfig, Settings
 from .model import Model
-from .plugin import get_plugin_namespace, load_plugin
+from .plugin import get_plugin_namespace, is_builtin_plugin, load_plugin
 from .scorer import Context, Score, Scorer
 from .utils import deep_merge_dicts, parse_study_direction, print
 
@@ -159,6 +159,22 @@ class Evaluator:
 
         return merged_settings
 
+    def all_scorers_reproducible(self) -> bool:
+        """
+        Returns True if all scorers are reproducible,
+        False if not.
+        """
+        return all(entry.scorer.reproducible for entry in self._scorer_entries)
+
+    def all_scorers_builtin(self) -> bool:
+        """
+        Returns True if all scorers are built-in,
+        i.e included in Heretic by default.
+        """
+        return all(
+            is_builtin_plugin(entry.config.plugin) for entry in self._scorer_entries
+        )
+
     def get_scores(self) -> list[tuple[str, Score]]:
         """
         Run all scorers and return their scores and names

src/heretic/main.py

@@ -72,7 +72,6 @@ from .analyzer import Analyzer
 from .config import ExportStrategy, QuantizationMethod
 from .evaluator import Evaluator
 from .model import AbliterationParameters, Model, get_model_class
-from .plugin import is_builtin_plugin
 from .reproduce import (
     check_environment,
     collect_reproducibles,
@@ -1129,10 +1128,8 @@ def run():
                                     and specification.commit is not None
                                     for specification in dataset_specifications
                                 )
-                                and all(
+                                and evaluator.all_scorers_reproducible()
-                                    is_builtin_plugin(scorer.plugin)
+                                and evaluator.all_scorers_builtin()
-                                    for scorer in settings.scorers
-                                )
                                 and not reproduction_mode
                             )
 

src/heretic/plugin.py

@@ -194,6 +194,22 @@ class Plugin:
       an instance as `settings`.
     """
 
+    @property
+    def reproducible(self) -> bool:
+        """
+        Whether runs using this plugin can be reproduced bit-for-bit.
+
+        Set to False when the plugin's behavior is not deterministic or depends on
+        state outside the pinned config, for example:
+        - It calls an external service (e.g. an LLM judge over the OpenAI API).
+        - It reads credentials or config from the environment (env vars, files).
+        - It is otherwise non-deterministic (network, wall-clock, unseeded RNG).
+
+        Defaults to False; override to True in your plugin class if any of the
+        above DO NOT apply.
+        """
+        return False
+
     def __init__(
         self, *, heretic_settings: HereticSettings, settings: BaseModel | None = None
     ):

src/heretic/scorers/keyword_rate.py

@@ -74,6 +74,10 @@ class KeywordRate(Scorer):
 
     settings: Settings
 
+    @property
+    def reproducible(self) -> bool:
+        return True
+
     @property
     def score_name(self) -> str:
         return "Keywords"

src/heretic/scorers/kl_divergence.py

@@ -31,6 +31,10 @@ class KLDivergence(Scorer):
 
     settings: Settings
 
+    @property
+    def reproducible(self) -> bool:
+        return True
+
     @property
     def score_name(self) -> str:
         return "KL divergence"