diff --git a/src/heretic/evaluator.py b/src/heretic/evaluator.py
index 3c6aba7..dbfbedd 100644
--- a/src/heretic/evaluator.py
+++ b/src/heretic/evaluator.py
@@ -9,7 +9,7 @@ from pydantic import BaseModel
 
 from .config import DatasetSpecification, ScorerConfig, Settings
 from .model import Model
-from .plugin import get_plugin_namespace, load_plugin
+from .plugin import get_plugin_namespace, is_builtin_plugin, load_plugin
 from .scorer import Context, Score, Scorer
 from .utils import deep_merge_dicts, parse_study_direction, print
 
@@ -159,6 +159,22 @@ class Evaluator:
 
         return merged_settings
 
+    def all_scorers_reproducible(self) -> bool:
+        """
+        Returns True if all scorers are reproducible,
+        False if not.
+        """
+        return all(entry.scorer.reproducible for entry in self._scorer_entries)
+
+    def all_scorers_builtin(self) -> bool:
+        """
+        Returns True if all scorers are built-in,
+        i.e included in Heretic by default.
+        """
+        return all(
+            is_builtin_plugin(entry.config.plugin) for entry in self._scorer_entries
+        )
+
     def get_scores(self) -> list[tuple[str, Score]]:
         """
         Run all scorers and return their scores and names
diff --git a/src/heretic/main.py b/src/heretic/main.py
index 10c6584..fbc6cd5 100644
--- a/src/heretic/main.py
+++ b/src/heretic/main.py
@@ -72,7 +72,6 @@ from .analyzer import Analyzer
 from .config import ExportStrategy, QuantizationMethod
 from .evaluator import Evaluator
 from .model import AbliterationParameters, Model, get_model_class
-from .plugin import is_builtin_plugin
 from .reproduce import (
     check_environment,
     collect_reproducibles,
@@ -1129,10 +1128,8 @@ def run():
                                     and specification.commit is not None
                                     for specification in dataset_specifications
                                 )
-                                and all(
-                                    is_builtin_plugin(scorer.plugin)
-                                    for scorer in settings.scorers
-                                )
+                                and evaluator.all_scorers_reproducible()
+                                and evaluator.all_scorers_builtin()
                                 and not reproduction_mode
                             )
 
diff --git a/src/heretic/plugin.py b/src/heretic/plugin.py
index 411c7b1..4b57fbf 100644
--- a/src/heretic/plugin.py
+++ b/src/heretic/plugin.py
@@ -194,6 +194,22 @@ class Plugin:
       an instance as `settings`.
     """
 
+    @property
+    def reproducible(self) -> bool:
+        """
+        Whether runs using this plugin can be reproduced bit-for-bit.
+
+        Set to False when the plugin's behavior is not deterministic or depends on
+        state outside the pinned config, for example:
+        - It calls an external service (e.g. an LLM judge over the OpenAI API).
+        - It reads credentials or config from the environment (env vars, files).
+        - It is otherwise non-deterministic (network, wall-clock, unseeded RNG).
+
+        Defaults to False; override to True in your plugin class if any of the
+        above DO NOT apply.
+        """
+        return False
+
     def __init__(
         self, *, heretic_settings: HereticSettings, settings: BaseModel | None = None
     ):
diff --git a/src/heretic/scorers/keyword_rate.py b/src/heretic/scorers/keyword_rate.py
index 0743421..4e6ffed 100644
--- a/src/heretic/scorers/keyword_rate.py
+++ b/src/heretic/scorers/keyword_rate.py
@@ -74,6 +74,10 @@ class KeywordRate(Scorer):
 
     settings: Settings
 
+    @property
+    def reproducible(self) -> bool:
+        return True
+
     @property
     def score_name(self) -> str:
         return "Keywords"
diff --git a/src/heretic/scorers/kl_divergence.py b/src/heretic/scorers/kl_divergence.py
index 319d31f..a3b97ac 100644
--- a/src/heretic/scorers/kl_divergence.py
+++ b/src/heretic/scorers/kl_divergence.py
@@ -31,6 +31,10 @@ class KLDivergence(Scorer):
 
     settings: Settings
 
+    @property
+    def reproducible(self) -> bool:
+        return True
+
     @property
     def score_name(self) -> str:
         return "KL divergence"