vh/heretic
1
0
Fork 0
mirror of https://github.com/p-e-w/heretic.git synced 2026-09-25 13:31:33 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat: add benchmark scorer (#444)

Browse Source 
* fix: improve print output of scorers

* feat: add benchmark scorer
This commit is contained in:
Philipp Emanuel Weidmann
2026-09-03 17:49:46 +05:30
committed by GitHub
parent c7a44f0db7
commit 95dda4c4db
11 changed files with 113 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/qwen2.5/config.toml
-6
View File
@@ -9,7 +9,6 @@ print_debug_information = true
batch_size = 2
max_response_length = 10
kl_divergence_target = 0
n_trials = 2
n_startup_trials = 1
@@ -21,11 +20,6 @@ save_directory = "model"
row_normalization = "pre"
scorers = [
{ plugin = "heretic.scorers.keyword_rate.KeywordRate", optimization = "minimize" },
{ plugin = "heretic.scorers.kl_divergence.KLDivergence", optimization = "minimize" },
]
[good_prompts]
dataset = "mlabonne/harmless_alpaca"
commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"