mirror of
https://github.com/p-e-w/heretic.git
synced 2026-10-01 08:21:29 -07:00
662e4ba27e
* feat: add modifier base class * feat: re-implement abliteration as a modifier plugin * fix: adjust good/bad prompts hack to match tests * feat: support dataset specifications containing multiple individual datasets * feat: re-implement ARA as a modifier plugin Arbitrary-Rank Ablation (ARA) (Weidmann 2026) was originally introduced by @p-e-w in #211 Co-authored-by: kabachuha <artemkhrapov2001@yandex.ru> Co-authored-by: joninco <joninco@bullpoint.org> Co-authored-by: Ashar <coder3101@users.noreply.github.com> * feat: add tests for ARA * feat: reduce default number of trials * ci: remove broken "semantic-pull-request" workflow * ci: add yet another alternative model hash --------- Co-authored-by: kabachuha <artemkhrapov2001@yandex.ru> Co-authored-by: joninco <joninco@bullpoint.org> Co-authored-by: Ashar <coder3101@users.noreply.github.com>
161 lines
3.2 KiB
TOML
161 lines
3.2 KiB
TOML
# Rename this file to config.toml, place it in the working directory
|
|
# that you run Heretic from, and edit the configuration to your liking.
|
|
|
|
max_response_length = 300
|
|
|
|
system_prompt = "You are a professional writer."
|
|
|
|
[scorer.KeywordRate]
|
|
score_name = "Responses with slop"
|
|
|
|
keyword_markers = [
|
|
"Eldoria",
|
|
"Lumina",
|
|
"ethereal",
|
|
"thick with",
|
|
"celestial",
|
|
"radiant",
|
|
"black as",
|
|
"despair",
|
|
"crimson",
|
|
"resplendent",
|
|
"unravel",
|
|
"belied",
|
|
"velvet",
|
|
"moonless",
|
|
"moonlit",
|
|
"entangled",
|
|
"twilight",
|
|
"forever",
|
|
"first kiss",
|
|
"gasp",
|
|
"whisper",
|
|
"hue",
|
|
"symphony",
|
|
"scarcely believe",
|
|
"gilded",
|
|
"hummed",
|
|
"abuzz",
|
|
"perpetually",
|
|
"scent",
|
|
"perfume",
|
|
"neon lights",
|
|
"kaleidoscopic",
|
|
"adrift",
|
|
"sultry",
|
|
"melancholic",
|
|
"stark contrast",
|
|
"inky",
|
|
"coy",
|
|
"vast",
|
|
"purr",
|
|
"radiant",
|
|
"beacon",
|
|
"a thousand ships",
|
|
"tapestry",
|
|
"bustling",
|
|
"abyss",
|
|
"gnarled",
|
|
"tremble",
|
|
"trembling",
|
|
"profound",
|
|
"terrible",
|
|
"ancient",
|
|
"sapphire",
|
|
"ruby",
|
|
"emerald",
|
|
"diamond",
|
|
"stolen",
|
|
"promise",
|
|
"the air was",
|
|
"obsidian",
|
|
"gleaming with",
|
|
"faintest hint",
|
|
"trepidation",
|
|
"sun-kissed",
|
|
"azure",
|
|
"deep",
|
|
"beloved",
|
|
"cosmos",
|
|
"devoid",
|
|
"soft chime",
|
|
"echo",
|
|
"palpable",
|
|
"blossom",
|
|
"adrift",
|
|
"faint",
|
|
"emerged",
|
|
"shiver",
|
|
"spine",
|
|
"hairs on the back",
|
|
"cinematic",
|
|
"specter",
|
|
"golden",
|
|
"inescapable",
|
|
"sentinel",
|
|
"flicker",
|
|
"testament",
|
|
"embodiment",
|
|
"etched with",
|
|
"rise and fall",
|
|
"the very air",
|
|
"slither",
|
|
"a pang of",
|
|
"eternal",
|
|
"eternity",
|
|
"veil of",
|
|
"painting the",
|
|
"bathed in",
|
|
"boundless",
|
|
"stretched out",
|
|
"beneath",
|
|
"lullaby",
|
|
"unsuspecting",
|
|
"handsome",
|
|
"defied the very",
|
|
"barely above",
|
|
"never-ending",
|
|
"caress",
|
|
"realm",
|
|
"fiery",
|
|
"raven",
|
|
"twin pools",
|
|
"gloaming",
|
|
"grimy",
|
|
"labyrinth",
|
|
"the very notion",
|
|
"something...",
|
|
"the halls of",
|
|
"conflagration of",
|
|
"shattered like",
|
|
"as dark as",
|
|
"yearned for",
|
|
"unyielding",
|
|
"lifetime",
|
|
"ensnared",
|
|
]
|
|
|
|
[scorer.KeywordRate.prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[1000:1100]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below.\n\nWriting prompt:"
|
|
|
|
[scorer.KLDivergence.prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[1000:1100]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below. Avoid literary cliches, purple prose, and flowery language.\n\nWriting prompt:"
|
|
|
|
[modifier.ARA.good_prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[:500]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below. Avoid literary cliches, purple prose, and flowery language.\n\nWriting prompt:"
|
|
|
|
[modifier.ARA.bad_prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[:500]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below. Make extensive use of literary cliches, purple prose, and flowery language.\n\nWriting prompt:"
|