mirror of
https://github.com/p-e-w/heretic.git
synced 2026-10-01 16:31:28 -07:00
feat: modifier plugins + ARA (#446)
* feat: add modifier base class * feat: re-implement abliteration as a modifier plugin * fix: adjust good/bad prompts hack to match tests * feat: support dataset specifications containing multiple individual datasets * feat: re-implement ARA as a modifier plugin Arbitrary-Rank Ablation (ARA) (Weidmann 2026) was originally introduced by @p-e-w in #211 Co-authored-by: kabachuha <artemkhrapov2001@yandex.ru> Co-authored-by: joninco <joninco@bullpoint.org> Co-authored-by: Ashar <coder3101@users.noreply.github.com> * feat: add tests for ARA * feat: reduce default number of trials * ci: remove broken "semantic-pull-request" workflow * ci: add yet another alternative model hash --------- Co-authored-by: kabachuha <artemkhrapov2001@yandex.ru> Co-authored-by: joninco <joninco@bullpoint.org> Co-authored-by: Ashar <coder3101@users.noreply.github.com>
This commit is contained in:
co-authored by
kabachuha
joninco
Ashar
parent
71e6d5eb38
commit
662e4ba27e
+10
-16
@@ -3,22 +3,6 @@
|
||||
|
||||
max_response_length = 300
|
||||
|
||||
residual_plot_title = "PaCMAP Projection of Residuals for Serious/Humorous Prompts"
|
||||
|
||||
[good_prompts]
|
||||
dataset = "mlabonne/harmless_alpaca"
|
||||
split = "train[:400]"
|
||||
column = "text"
|
||||
residual_plot_label = "Serious prompts"
|
||||
residual_plot_color = "royalblue"
|
||||
|
||||
[bad_prompts]
|
||||
dataset = "UnstableLlama/jokes"
|
||||
split = "train[:200]"
|
||||
column = "text"
|
||||
residual_plot_label = "Humorous prompts"
|
||||
residual_plot_color = "darkorange"
|
||||
|
||||
[scorer.KeywordRate]
|
||||
score_name = "Responses with humor"
|
||||
|
||||
@@ -70,3 +54,13 @@ column = "text"
|
||||
dataset = "mlabonne/harmless_alpaca"
|
||||
split = "test[:100]"
|
||||
column = "text"
|
||||
|
||||
[modifier.ARA.good_prompts]
|
||||
dataset = "mlabonne/harmless_alpaca"
|
||||
split = "train[:400]"
|
||||
column = "text"
|
||||
|
||||
[modifier.ARA.bad_prompts]
|
||||
dataset = "UnstableLlama/jokes"
|
||||
split = "train[:200]"
|
||||
column = "text"
|
||||
|
||||
Reference in New Issue
Block a user