feat: headless operation + end-to-end tests (#392)

* fix: remove notebook input shims

Closes #280

* feat: support headless operation (no interactive input)

* fix: prevent infinite loops

* feat: add end-to-end tests

* ci: run tests in CI

* ci: fix test output ordering

* fix: replace home-cooked `set_seed` function with Transformers builtin

* feat: print PyTorch config when running tests

* feat: print additional information

* experiment: try to standardize test environment

* fix: revert environment changes

* feat: support multiple valid hashes for each output file

* feat: add test output hashes for CI

* feat: add test output hashes for CI (alternative environment)

* feat: add hashes for Windows (#394)

* fix: Hash on windows

* trigger ci

* fix: prefer .yaml (used widely than .toml for model configs)

* use removeprefix

* docs: restore commet

* use removeprefix again

* tests: Add windows hash files for all test models

* trigger ci

* fix: minor cleanup

* clean merge mismatch

* remove unnecessary CRLF replace, now that we support more SUMS files

* fix: use binary mode for hashes everywhere

---------

Co-authored-by: Vinay Umrethe <umrethevinay@gmail.com>

tests/mistral-3/SHA256SUMS.ci

@@ -0,0 +1,7 @@
+39f03c383413f531fd302c06c7e982ad98c83f0657a8339ae25478ccb81fdcda *chat_template.jinja
+f69f84977a47c8fea9ce9fc26b7de379216cb01146ea726a87996d3554cfcd19 *config.json
+34dfa6012ca9ac5f57e5521d8dbaecbc7ab7f7ab0fd96ec020b543aab5f265d9 *generation_config.json
+876c6691eb85e3e5e11771e589529830fb454ab26344e1271ae550661e312b50 *model.safetensors
+84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491 *processor_config.json
+c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f *tokenizer.json
+7b29c843c0043622d28fd4638451cbb0a609d99a0762ffbff3b92b4b2fee4d94 *tokenizer_config.json

tests/mistral-3/SHA256SUMS.ci2

@@ -0,0 +1,7 @@
+39f03c383413f531fd302c06c7e982ad98c83f0657a8339ae25478ccb81fdcda *chat_template.jinja
+f69f84977a47c8fea9ce9fc26b7de379216cb01146ea726a87996d3554cfcd19 *config.json
+34dfa6012ca9ac5f57e5521d8dbaecbc7ab7f7ab0fd96ec020b543aab5f265d9 *generation_config.json
+6febb813086f253e5ec0fcda02fdfc849c551a7dba54681b37ac5bc402e4eed6 *model.safetensors
+84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491 *processor_config.json
+c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f *tokenizer.json
+7b29c843c0043622d28fd4638451cbb0a609d99a0762ffbff3b92b4b2fee4d94 *tokenizer_config.json

tests/mistral-3/SHA256SUMS.linux

@@ -0,0 +1,7 @@
+39f03c383413f531fd302c06c7e982ad98c83f0657a8339ae25478ccb81fdcda *chat_template.jinja
+f69f84977a47c8fea9ce9fc26b7de379216cb01146ea726a87996d3554cfcd19 *config.json
+34dfa6012ca9ac5f57e5521d8dbaecbc7ab7f7ab0fd96ec020b543aab5f265d9 *generation_config.json
+29aff97d5633dead9e1ccd29a2cc153b4b7431d22f63c8d6cf60bc6547681cc9 *model.safetensors
+84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491 *processor_config.json
+c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f *tokenizer.json
+7b29c843c0043622d28fd4638451cbb0a609d99a0762ffbff3b92b4b2fee4d94 *tokenizer_config.json

tests/mistral-3/SHA256SUMS.windows

@@ -0,0 +1,7 @@
+72f84af4ea36b82409c35e31b584361534305ef7c0d90fce20d0dc38a7efead8 *chat_template.jinja
+e4c5278b361c57621253c27a2c3db358e1580aec8a14be8e19d4420a224137cf *config.json
+8dde85c000ae807be907421465826c7c63a39f6acf6d04a5a84efaf116ed4ef7 *generation_config.json
+29aff97d5633dead9e1ccd29a2cc153b4b7431d22f63c8d6cf60bc6547681cc9 *model.safetensors
+20e7a6dcde0a6f60ea3b4fb08f6f7afa62532dda93a3111e28384ba5150575f9 *processor_config.json
+c3a8d92e371b92a2cd6e678e31ebc27d0235e929a51fbf290f74742b341fa96f *tokenizer.json
+60a8042e29b4b20e884e48375aa1b9ac0025547371d50e60f6d55e6a9675e868 *tokenizer_config.json

tests/mistral-3/config.toml

@@ -0,0 +1,41 @@
+model = "tiny-random/mistral-3"
+model_commit = "931aa2e5c9668fc3679e56aa44972fe18597d55d"
+
+seed = 12345
+print_debug_information = true
+
+batch_size = 2
+max_response_length = 10
+kl_divergence_target = 0
+n_trials = 2
+n_startup_trials = 1
+
+export_strategy = "merge"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[good_evaluation_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[bad_evaluation_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"