diff --git a/README.md b/README.md index 52659e3..71ac3d7 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ models with Heretic. Prepare a Python 3.10+ environment with PyTorch 2.2+ installed as appropriate for your hardware. Then run: -``` +```sh pip install -U heretic-llm heretic Qwen/Qwen3-4B-Instruct-2507 ``` @@ -134,7 +134,7 @@ provides features designed to support research into the semantics of model inter (interpretability). To use those features, you need to install Heretic with the optional `research` extra: -``` +```sh pip install -U heretic-llm[research] ``` diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..8d107af --- /dev/null +++ b/tests/README.md @@ -0,0 +1,17 @@ +Run the tests with + +```sh +uv run run_tests.py +``` + +To update the hashes after a logic change, run the tests, then execute + +```sh +cd TEST_DIR/model +sha256sum * > ../SHA256SUMS.LABEL +``` + +where `LABEL` describes the type of system you are running the tests on. +Since PyTorch does not guarantee exact cross-system reproducibility regardless of configuration, +multiple valid hashes can be provided for each output file. The above update must be performed +for each `TEST_DIR` and on each type of system. diff --git a/tests/gemma-4e/SHA256SUMS.linux b/tests/gemma-4e/SHA256SUMS.linux new file mode 100644 index 0000000..35a6a39 --- /dev/null +++ b/tests/gemma-4e/SHA256SUMS.linux @@ -0,0 +1,7 @@ +2f1b4d75d067bae3fe44e676721c7f077d243bc007156cb9c2f8b5836613d082 chat_template.jinja +ca80080dfa4ec6ba87152fa2b9afe70b90c400e5c4b1d6bdc3aa3114467ca68f config.json +70070bac883cf9c39b5992450d6b23cd160eaf33099e24c654e0359d2f87c760 generation_config.json +effe36925f85ecb1e29bba84501a456bb49df21e4047be8b7ea3f6f88181fb65 model.safetensors +32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c processor_config.json +cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f tokenizer.json +a1bab8c81ed15fa6ce912ec993c66cb49392e0487fb1ea5f5f11ea3618683627 tokenizer_config.json diff --git a/tests/gemma-4e/config.toml b/tests/gemma-4e/config.toml new file mode 100644 index 0000000..3a583cd --- /dev/null +++ b/tests/gemma-4e/config.toml @@ -0,0 +1,41 @@ +model = "tiny-random/gemma-4e" +model_commit = "3a207ada2c2cd95e9671942e84cf47ea58f0f6af" + +seed = 12345 +print_debug_information = true + +batch_size = 2 +max_response_length = 10 +kl_divergence_target = 0 +n_trials = 2 +n_startup_trials = 1 + +export_strategy = "merge" +checkpoint_action = "restart" +trial_index = 0 +model_action = "save" +save_directory = "model" + +[good_prompts] +dataset = "mlabonne/harmless_alpaca" +commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f" +split = "train[:5]" +column = "text" + +[bad_prompts] +dataset = "mlabonne/harmful_behaviors" +commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7" +split = "train[:5]" +column = "text" + +[good_evaluation_prompts] +dataset = "mlabonne/harmless_alpaca" +commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f" +split = "test[:5]" +column = "text" + +[bad_evaluation_prompts] +dataset = "mlabonne/harmful_behaviors" +commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7" +split = "test[:5]" +column = "text" diff --git a/tests/mistral-3/SHA256SUMS b/tests/mistral-3/SHA256SUMS.linux similarity index 89% rename from tests/mistral-3/SHA256SUMS rename to tests/mistral-3/SHA256SUMS.linux index 62bf8c2..48d633f 100644 --- a/tests/mistral-3/SHA256SUMS +++ b/tests/mistral-3/SHA256SUMS.linux @@ -1,7 +1,7 @@ 39f03c383413f531fd302c06c7e982ad98c83f0657a8339ae25478ccb81fdcda chat_template.jinja f69f84977a47c8fea9ce9fc26b7de379216cb01146ea726a87996d3554cfcd19 config.json 34dfa6012ca9ac5f57e5521d8dbaecbc7ab7f7ab0fd96ec020b543aab5f265d9 generation_config.json -26b0435167dba8138d21eff9c511e909d501361805f946828de0fd19ada434dc model.safetensors +29aff97d5633dead9e1ccd29a2cc153b4b7431d22f63c8d6cf60bc6547681cc9 model.safetensors 84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491 processor_config.json c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f tokenizer.json 7b29c843c0043622d28fd4638451cbb0a609d99a0762ffbff3b92b4b2fee4d94 tokenizer_config.json diff --git a/tests/qwen3.5-moe/SHA256SUMS.linux b/tests/qwen3.5-moe/SHA256SUMS.linux new file mode 100644 index 0000000..33f15d0 --- /dev/null +++ b/tests/qwen3.5-moe/SHA256SUMS.linux @@ -0,0 +1,7 @@ +a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715 chat_template.jinja +749b56d1b1e08081981169db6f2c44ab0be4fd6ebb452d15baafa5e09c21586a config.json +4625d1d64d41d1fa9dae7af4ba1e1d7e65a194073d4efa58acb266a916eaaa74 generation_config.json +5e0fb0ac724cf079b693fc76a515e60bc16de72c32b36c107b9f078061c4f2ef model.safetensors +01562eddd6f9e9ec4bc31656a3b7055284cafbf889acc6c4348dca431ae31f68 processor_config.json +87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4 tokenizer.json +2e31d1126e81bddf8d15c3f95260fb487b48c5131b24fcbb5bb9d2537e7afac0 tokenizer_config.json diff --git a/tests/qwen3.5-moe/config.toml b/tests/qwen3.5-moe/config.toml new file mode 100644 index 0000000..f708d0d --- /dev/null +++ b/tests/qwen3.5-moe/config.toml @@ -0,0 +1,41 @@ +model = "tiny-random/qwen3.5-moe" +model_commit = "2ebfa8d9717238c5dda927008104fa172a149050" + +seed = 12345 +print_debug_information = true + +batch_size = 2 +max_response_length = 10 +kl_divergence_target = 0 +n_trials = 2 +n_startup_trials = 1 + +export_strategy = "merge" +checkpoint_action = "restart" +trial_index = 0 +model_action = "save" +save_directory = "model" + +[good_prompts] +dataset = "mlabonne/harmless_alpaca" +commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f" +split = "train[:5]" +column = "text" + +[bad_prompts] +dataset = "mlabonne/harmful_behaviors" +commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7" +split = "train[:5]" +column = "text" + +[good_evaluation_prompts] +dataset = "mlabonne/harmless_alpaca" +commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f" +split = "test[:5]" +column = "text" + +[bad_evaluation_prompts] +dataset = "mlabonne/harmful_behaviors" +commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7" +split = "test[:5]" +column = "text" diff --git a/tests/run_tests.py b/tests/run_tests.py index c4d7dd9..ef57923 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -23,12 +23,14 @@ script_directory = Path(__file__).resolve().parent project_directory = script_directory.parent +tests_failed = False + for test_directory in script_directory.iterdir(): if test_directory.is_dir(): config_file = test_directory / "config.toml" - hash_file = test_directory / "SHA256SUMS" + hash_files = list(test_directory.glob("SHA256SUMS.*")) - if config_file.is_file() and hash_file.is_file(): + if config_file.is_file() and hash_files: print("#" * 50) print(f"Running test {test_directory.name}") print("#" * 50) @@ -49,25 +51,36 @@ for test_directory in script_directory.iterdir(): print() - # To update the hashes after a logic change, run the tests, then execute - # - # cd /model - # sha256sum * > ../SHA256SUMS + valid_hashes: dict[str, list[str]] = {} - with open(hash_file, "r", encoding="utf-8") as file: - for line in file: - if line.strip(): - original_sha256, filename = line.split() - sha256 = get_file_sha256(test_directory / "model" / filename) + for hash_file in hash_files: + with open(hash_file, "r", encoding="utf-8") as file: + for line in file: + if line.strip(): + sha256, filename = line.split() - if sha256.lower() != original_sha256.lower(): - sys.exit( - ( - f"Test {test_directory.name} has FAILED!\n" - f"Output file {filename} doesn't match.\n" - f"Expected hash: {original_sha256}\n" - f"Actual hash: {sha256}" - ) - ) + if filename not in valid_hashes: + valid_hashes[filename] = [] -print("All tests passed!") + valid_hashes[filename].append(sha256.lower()) + + for filename in valid_hashes: + sha256 = get_file_sha256(test_directory / "model" / filename) + + if sha256.lower() not in valid_hashes[filename]: + print( + ( + f"Test {test_directory.name} has FAILED!\n" + f"Output file {filename} doesn't match any valid hash.\n\n" + f"Valid hashes:\n" + f"{chr(10).join(valid_hashes[filename])}\n\n" + f"Actual hash:\n" + f"{sha256}\n" + ) + ) + tests_failed = True + +if tests_failed: + sys.exit("Tests failed.") +else: + print("All tests passed.")