feat: support multiple valid hashes for each output file

README.md

@@ -86,7 +86,7 @@ models with Heretic.
 Prepare a Python 3.10+ environment with PyTorch 2.2+ installed as appropriate
 for your hardware. Then run:
 
-```
+```sh
 pip install -U heretic-llm
 heretic Qwen/Qwen3-4B-Instruct-2507
 ```
@@ -134,7 +134,7 @@ provides features designed to support research into the semantics of model inter
 (interpretability). To use those features, you need to install Heretic with the
 optional `research` extra:
 
-```
+```sh
 pip install -U heretic-llm[research]
 ```
 

tests/README.md

@@ -0,0 +1,17 @@
+Run the tests with
+
+```sh
+uv run run_tests.py
+```
+
+To update the hashes after a logic change, run the tests, then execute
+
+```sh
+cd TEST_DIR/model
+sha256sum * > ../SHA256SUMS.LABEL
+```
+
+where `LABEL` describes the type of system you are running the tests on.
+Since PyTorch does not guarantee exact cross-system reproducibility regardless of configuration,
+multiple valid hashes can be provided for each output file. The above update must be performed
+for each `TEST_DIR` and on each type of system.

tests/gemma-4e/SHA256SUMS.linux

@@ -0,0 +1,7 @@
+2f1b4d75d067bae3fe44e676721c7f077d243bc007156cb9c2f8b5836613d082  chat_template.jinja
+ca80080dfa4ec6ba87152fa2b9afe70b90c400e5c4b1d6bdc3aa3114467ca68f  config.json
+70070bac883cf9c39b5992450d6b23cd160eaf33099e24c654e0359d2f87c760  generation_config.json
+effe36925f85ecb1e29bba84501a456bb49df21e4047be8b7ea3f6f88181fb65  model.safetensors
+32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c  processor_config.json
+cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f  tokenizer.json
+a1bab8c81ed15fa6ce912ec993c66cb49392e0487fb1ea5f5f11ea3618683627  tokenizer_config.json

tests/gemma-4e/config.toml

@@ -0,0 +1,41 @@
+model = "tiny-random/gemma-4e"
+model_commit = "3a207ada2c2cd95e9671942e84cf47ea58f0f6af"
+
+seed = 12345
+print_debug_information = true
+
+batch_size = 2
+max_response_length = 10
+kl_divergence_target = 0
+n_trials = 2
+n_startup_trials = 1
+
+export_strategy = "merge"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[good_evaluation_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[bad_evaluation_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"

tests/mistral-3/SHA256SUMS.linux

@@ -1,7 +1,7 @@
 39f03c383413f531fd302c06c7e982ad98c83f0657a8339ae25478ccb81fdcda  chat_template.jinja
 f69f84977a47c8fea9ce9fc26b7de379216cb01146ea726a87996d3554cfcd19  config.json
 34dfa6012ca9ac5f57e5521d8dbaecbc7ab7f7ab0fd96ec020b543aab5f265d9  generation_config.json
-26b0435167dba8138d21eff9c511e909d501361805f946828de0fd19ada434dc  model.safetensors
+29aff97d5633dead9e1ccd29a2cc153b4b7431d22f63c8d6cf60bc6547681cc9  model.safetensors
 84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491  processor_config.json
 c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f  tokenizer.json
 7b29c843c0043622d28fd4638451cbb0a609d99a0762ffbff3b92b4b2fee4d94  tokenizer_config.json

tests/qwen3.5-moe/SHA256SUMS.linux

@@ -0,0 +1,7 @@
+a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715  chat_template.jinja
+749b56d1b1e08081981169db6f2c44ab0be4fd6ebb452d15baafa5e09c21586a  config.json
+4625d1d64d41d1fa9dae7af4ba1e1d7e65a194073d4efa58acb266a916eaaa74  generation_config.json
+5e0fb0ac724cf079b693fc76a515e60bc16de72c32b36c107b9f078061c4f2ef  model.safetensors
+01562eddd6f9e9ec4bc31656a3b7055284cafbf889acc6c4348dca431ae31f68  processor_config.json
+87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4  tokenizer.json
+2e31d1126e81bddf8d15c3f95260fb487b48c5131b24fcbb5bb9d2537e7afac0  tokenizer_config.json

tests/qwen3.5-moe/config.toml

@@ -0,0 +1,41 @@
+model = "tiny-random/qwen3.5-moe"
+model_commit = "2ebfa8d9717238c5dda927008104fa172a149050"
+
+seed = 12345
+print_debug_information = true
+
+batch_size = 2
+max_response_length = 10
+kl_divergence_target = 0
+n_trials = 2
+n_startup_trials = 1
+
+export_strategy = "merge"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[good_evaluation_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[bad_evaluation_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"

tests/run_tests.py

@@ -23,12 +23,14 @@ script_directory = Path(__file__).resolve().parent
 
 project_directory = script_directory.parent
 
+tests_failed = False
+
 for test_directory in script_directory.iterdir():
     if test_directory.is_dir():
         config_file = test_directory / "config.toml"
-        hash_file = test_directory / "SHA256SUMS"
+        hash_files = list(test_directory.glob("SHA256SUMS.*"))
 
-        if config_file.is_file() and hash_file.is_file():
+        if config_file.is_file() and hash_files:
             print("#" * 50)
             print(f"Running test {test_directory.name}")
             print("#" * 50)
@@ -49,25 +51,36 @@ for test_directory in script_directory.iterdir():
 
             print()
 
-            # To update the hashes after a logic change, run the tests, then execute
+            valid_hashes: dict[str, list[str]] = {}
-            #
-            # cd <test_dir>/model
-            # sha256sum * > ../SHA256SUMS
 
-            with open(hash_file, "r", encoding="utf-8") as file:
+            for hash_file in hash_files:
-                for line in file:
+                with open(hash_file, "r", encoding="utf-8") as file:
-                    if line.strip():
+                    for line in file:
-                        original_sha256, filename = line.split()
+                        if line.strip():
-                        sha256 = get_file_sha256(test_directory / "model" / filename)
+                            sha256, filename = line.split()
 
-                        if sha256.lower() != original_sha256.lower():
+                            if filename not in valid_hashes:
-                            sys.exit(
+                                valid_hashes[filename] = []
-                                (
-                                    f"Test {test_directory.name} has FAILED!\n"
-                                    f"Output file {filename} doesn't match.\n"
-                                    f"Expected hash: {original_sha256}\n"
-                                    f"Actual hash:   {sha256}"
-                                )
-                            )
 
-print("All tests passed!")
+                            valid_hashes[filename].append(sha256.lower())
+
+            for filename in valid_hashes:
+                sha256 = get_file_sha256(test_directory / "model" / filename)
+
+                if sha256.lower() not in valid_hashes[filename]:
+                    print(
+                        (
+                            f"Test {test_directory.name} has FAILED!\n"
+                            f"Output file {filename} doesn't match any valid hash.\n\n"
+                            f"Valid hashes:\n"
+                            f"{chr(10).join(valid_hashes[filename])}\n\n"
+                            f"Actual hash:\n"
+                            f"{sha256}\n"
+                        )
+                    )
+                    tests_failed = True
+
+if tests_failed:
+    sys.exit("Tests failed.")
+else:
+    print("All tests passed.")