feat: add end-to-end tests

tests/mistral-3/SHA256SUMS

@@ -0,0 +1,7 @@
+39f03c383413f531fd302c06c7e982ad98c83f0657a8339ae25478ccb81fdcda  chat_template.jinja
+f69f84977a47c8fea9ce9fc26b7de379216cb01146ea726a87996d3554cfcd19  config.json
+34dfa6012ca9ac5f57e5521d8dbaecbc7ab7f7ab0fd96ec020b543aab5f265d9  generation_config.json
+26b0435167dba8138d21eff9c511e909d501361805f946828de0fd19ada434dc  model.safetensors
+84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491  processor_config.json
+c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f  tokenizer.json
+7b29c843c0043622d28fd4638451cbb0a609d99a0762ffbff3b92b4b2fee4d94  tokenizer_config.json

tests/mistral-3/config.toml

@@ -0,0 +1,40 @@
+model = "tiny-random/mistral-3"
+model_commit = "931aa2e5c9668fc3679e56aa44972fe18597d55d"
+
+batch_size = 2
+max_response_length = 10
+kl_divergence_target = 0
+n_trials = 2
+n_startup_trials = 1
+
+seed = 12345
+
+export_strategy = "merge"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[good_evaluation_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[bad_evaluation_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"

tests/run_tests.py

@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025-2026  Philipp Emanuel Weidmann <pew@worldwidemann.com> + contributors
+
+import hashlib
+import subprocess
+import sys
+from pathlib import Path
+
+
+# TODO: Replace this with hashlib.file_digest when we drop support for Python 3.10.
+def get_file_sha256(file_path: str | Path) -> str:
+    hash = hashlib.sha256()
+
+    with open(file_path, "rb") as file:
+        # Read the file in 64 kB blocks.
+        for block in iter(lambda: file.read(65536), b""):
+            hash.update(block)
+
+    return hash.hexdigest()
+
+
+script_directory = Path(__file__).resolve().parent
+
+project_directory = script_directory.parent
+
+for test_directory in script_directory.iterdir():
+    if test_directory.is_dir():
+        config_file = test_directory / "config.toml"
+        hash_file = test_directory / "SHA256SUMS"
+
+        if config_file.is_file() and hash_file.is_file():
+            print("#" * 50)
+            print(f"Running test {test_directory.name}")
+            print("#" * 50)
+            print()
+
+            subprocess.run(
+                [
+                    "uv",
+                    "run",
+                    "--project",
+                    project_directory,
+                    "--directory",
+                    test_directory,
+                    "heretic",
+                ],
+                check=True,
+            )
+
+            print()
+
+            # To update the hashes after a logic change, run the tests, then execute
+            #
+            # cd <test_dir>/model
+            # sha256sum * > ../SHA256SUMS
+
+            with open(hash_file, "r", encoding="utf-8") as file:
+                for line in file:
+                    if line.strip():
+                        original_sha256, filename = line.split()
+                        sha256 = get_file_sha256(test_directory / "model" / filename)
+
+                        if sha256.lower() != original_sha256.lower():
+                            sys.exit(
+                                (
+                                    f"Test {test_directory.name} has FAILED!\n"
+                                    f"Output file {filename} doesn't match.\n"
+                                    f"Expected hash: {original_sha256}\n"
+                                    f"Actual hash:   {sha256}"
+                                )
+                            )
+
+print("All tests passed!")