Merge branch 'master' into benchmark-scorer

config.default.toml

@@ -91,8 +91,8 @@ residual_plot_style = "dark_background"
 # { plugin = <plugin>, optimization = <optimization>, instance_name = <optional> }
 # where <optimization> is one of "minimize", "maximize", "none" (do not optimize)
 scorers = [
-    { plugin = "heretic.scorers.keyword_rate.KeywordRate", optimization = "minimize" },
-    { plugin = "heretic.scorers.kl_divergence.KLDivergence", optimization = "minimize" },
+    { plugin = "heretic.scorers.keyword_rate.KeywordRate", optimization = "minimize"},
+    { plugin = "heretic.scorers.kl_divergence.KLDivergence", optimization = "minimize"},
 ]
 
 # Whether to adjust the residual directions so that only the component that is

config.piqa.toml

@@ -2,6 +2,6 @@
 # that you run Heretic from, and edit the configuration to your liking.
 
 scorers = [
-    { plugin = "heretic.scorers.keyword_rate.KeywordRate", optimization = "minimize" },
-    { plugin = "heretic.scorers.benchmark_score.BenchmarkScore", optimization = "maximize" },
+    { plugin = "heretic.scorers.keyword_rate.KeywordRate", optimization = "minimize"},
+    { plugin = "heretic.scorers.benchmark_score.BenchmarkScore", optimization = "maximize"},
 ]

src/heretic/modifier.py

@@ -1,53 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# Copyright (C) 2025-2026  Philipp Emanuel Weidmann <pew@worldwidemann.com> + contributors
-
-from abc import ABC, abstractmethod
-from typing import Generic, TypeVar
-
-from optuna import Trial
-from pydantic import BaseModel
-
-from heretic.plugin import Context, Plugin
-
-from .config import Settings as HereticSettings
-
-Parameters = TypeVar("Parameters")
-
-
-class Modifier(Generic[Parameters], Plugin, ABC):
-    """
-    Abstract base class for modifier plugins.
-
-    Modifiers modify models based on an implementation-dependent set of optimizable parameters.
-
-    Examples: Standard abliteration, ARA, SOMA, etc.
-    """
-
-    def __init__(
-        self,
-        heretic_settings: HereticSettings,
-        settings: BaseModel | None = None,
-    ) -> None:
-        super().__init__(heretic_settings=heretic_settings, settings=settings)
-
-    @abstractmethod
-    def suggest_parameters(self, ctx: Context, trial: Trial) -> Parameters:
-        """
-        Sample parameters for a trial using the trial's `suggest_*` methods,
-        collect them in an implementation-dependent parameters object, and
-        return that object.
-        """
-
-    @abstractmethod
-    def modify_model(self, ctx: Context, parameters: Parameters) -> None:
-        """
-        Modify the model (obtainable via `ctx.get_model()`)
-        according to the provided parameters.
-        """
-
-    @abstractmethod
-    def reset_model(self, ctx: Context) -> None:
-        """
-        Reset the model (obtainable via `ctx.get_model()`),
-        undoing any changes made by `modify_model`.
-        """

src/heretic/plugin.py

@@ -149,8 +149,12 @@ def load_plugin(
 
 class Context:
     """
-    Runtime context passed to plugins.
-    Acts as a quasi-API for plugins to access Heretic functionality.
+    Runtime context passed to plugins
+
+    Provides plugin-safe access to the model.
+
+    Plugins must use `get_responses(...)`, `get_logits(...)`, etc.
+    Direct access to the underlying Model is intentionally not exposed.
     """
 
     def __init__(self, settings: HereticSettings, model: Model) -> None:
@@ -176,13 +180,6 @@ class Context:
     def get_residuals(self, prompts: list[Prompt]) -> Tensor:
         return self._model.get_residuals_batched(prompts)
 
-    def get_model(self) -> Model:
-        """
-        Prefer managed methods (`get_responses` etc.) unless you
-        actually need access to the model object.
-        """
-        return self._model
-
     def load_prompts(self, specification: DatasetSpecification) -> list[Prompt]:
         return load_prompts(self._settings, specification)
 
@@ -214,11 +211,8 @@ class Plugin:
         return False
 
     def __init__(
-        self,
-        *,
-        heretic_settings: HereticSettings,
-        settings: BaseModel | None = None,
-    ) -> None:
+        self, *, heretic_settings: HereticSettings, settings: BaseModel | None = None
+    ):
         # Plugins that declare a settings schema should always receive
         # validated plugin settings from the evaluator.
         settings_model = self.__class__.get_settings_model()

src/heretic/scorer.py

@@ -32,7 +32,7 @@ class Scorer(Plugin, ABC):
 
     Scorers evaluate model behavior and return a Score.
 
-    Examples: Counting refusals, measuring KL divergence, etc.
+    Example: counting refusals, measuring KL divergence, etc.
     """
 
     @property
@@ -47,7 +47,7 @@ class Scorer(Plugin, ABC):
         self,
         heretic_settings: HereticSettings,
         settings: BaseModel | None = None,
-    ) -> None:
+    ):
         super().__init__(heretic_settings=heretic_settings, settings=settings)
 
     @abstractmethod

src/heretic/scorers/benchmark_score.py

@@ -41,11 +41,9 @@ class BenchmarkScore(Scorer):
         return self.settings.score_name
 
     def init(self, ctx: Context) -> None:
-        model = ctx.get_model()
-
         self.hflm = HFLM(
-            pretrained=model.model,  # ty:ignore[invalid-argument-type]
-            tokenizer=model.tokenizer,  # ty:ignore[invalid-argument-type]
+            pretrained=ctx._model.model,  # ty:ignore[invalid-argument-type]
+            tokenizer=ctx._model.tokenizer,  # ty:ignore[invalid-argument-type]
             batch_size="auto",
         )
 
@@ -54,9 +52,8 @@ class BenchmarkScore(Scorer):
         # then update its internal model every time we calculate the score,
         # is to get the benefits of batch size caching while allowing for
         # model reloads, e.g. when using --evaluate-model.
-        model = ctx.get_model()
-        self.hflm.pretrained = model.model
-        self.hflm._model = model.model
+        self.hflm.pretrained = ctx._model.model
+        self.hflm._model = ctx._model.model
 
         results = lm_eval.simple_evaluate(
             model=self.hflm,

src/heretic/utils.py

@@ -43,6 +43,8 @@ T = TypeVar("T")
 
 print = Console(highlight=False).print
 
+T = TypeVar("T")
+
 
 def deep_merge_dicts(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
     """