feat: headless operation + end-to-end tests (#392)

* fix: remove notebook input shims

Closes #280

* feat: support headless operation (no interactive input)

* fix: prevent infinite loops

* feat: add end-to-end tests

* ci: run tests in CI

* ci: fix test output ordering

* fix: replace home-cooked `set_seed` function with Transformers builtin

* feat: print PyTorch config when running tests

* feat: print additional information

* experiment: try to standardize test environment

* fix: revert environment changes

* feat: support multiple valid hashes for each output file

* feat: add test output hashes for CI

* feat: add test output hashes for CI (alternative environment)

* feat: add hashes for Windows (#394)

* fix: Hash on windows

* trigger ci

* fix: prefer .yaml (used widely than .toml for model configs)

* use removeprefix

* docs: restore commet

* use removeprefix again

* tests: Add windows hash files for all test models

* trigger ci

* fix: minor cleanup

* clean merge mismatch

* remove unnecessary CRLF replace, now that we support more SUMS files

* fix: use binary mode for hashes everywhere

---------

Co-authored-by: Vinay Umrethe <umrethevinay@gmail.com>

tests/gemma-4e/SHA256SUMS.ci

@@ -0,0 +1,7 @@
+2f1b4d75d067bae3fe44e676721c7f077d243bc007156cb9c2f8b5836613d082 *chat_template.jinja
+ca80080dfa4ec6ba87152fa2b9afe70b90c400e5c4b1d6bdc3aa3114467ca68f *config.json
+70070bac883cf9c39b5992450d6b23cd160eaf33099e24c654e0359d2f87c760 *generation_config.json
+f3f4ec19504f182486459cf4e255ece265c25f827840d63b6a9d4058b8e4877a *model.safetensors
+32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c *processor_config.json
+cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f *tokenizer.json
+a1bab8c81ed15fa6ce912ec993c66cb49392e0487fb1ea5f5f11ea3618683627 *tokenizer_config.json

tests/gemma-4e/SHA256SUMS.ci2

@@ -0,0 +1,7 @@
+2f1b4d75d067bae3fe44e676721c7f077d243bc007156cb9c2f8b5836613d082 *chat_template.jinja
+ca80080dfa4ec6ba87152fa2b9afe70b90c400e5c4b1d6bdc3aa3114467ca68f *config.json
+70070bac883cf9c39b5992450d6b23cd160eaf33099e24c654e0359d2f87c760 *generation_config.json
+53c4ee891dce23c0ac85bebc2c4d48301469750fafbb3e6e024c15786d94db8b *model.safetensors
+32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c *processor_config.json
+cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f *tokenizer.json
+a1bab8c81ed15fa6ce912ec993c66cb49392e0487fb1ea5f5f11ea3618683627 *tokenizer_config.json

tests/gemma-4e/SHA256SUMS.linux

@@ -0,0 +1,7 @@
+2f1b4d75d067bae3fe44e676721c7f077d243bc007156cb9c2f8b5836613d082 *chat_template.jinja
+ca80080dfa4ec6ba87152fa2b9afe70b90c400e5c4b1d6bdc3aa3114467ca68f *config.json
+70070bac883cf9c39b5992450d6b23cd160eaf33099e24c654e0359d2f87c760 *generation_config.json
+effe36925f85ecb1e29bba84501a456bb49df21e4047be8b7ea3f6f88181fb65 *model.safetensors
+32bdf45d2ad4cc29a0822ddd157a182de76644f0419a6228d151495256e9813c *processor_config.json
+cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f *tokenizer.json
+a1bab8c81ed15fa6ce912ec993c66cb49392e0487fb1ea5f5f11ea3618683627 *tokenizer_config.json

tests/gemma-4e/SHA256SUMS.windows

@@ -0,0 +1,7 @@
+b16d3228a775c549ba97af41233a54e9de8dd2b65250f78346661d18b936a8b5 *chat_template.jinja
+0094ad598a8043f84d82ad5c886547bca1d1d7f302d82f1491f83d388e89acd4 *config.json
+1a019c5d688d54cf01318eab88cb4345dfa52135eb1d83c2f54125469eb88d5c *generation_config.json
+effe36925f85ecb1e29bba84501a456bb49df21e4047be8b7ea3f6f88181fb65 *model.safetensors
+24d00232e58cfa179fe8b3911c788d4aad9a6279d778ebe4c72e82623b6197f9 *processor_config.json
+cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f *tokenizer.json
+8044bbbddaee8dc47e6b5660e013ba92224d4a5392b2939c59699aa0105f5c8b *tokenizer_config.json

tests/gemma-4e/config.toml

@@ -0,0 +1,41 @@
+model = "tiny-random/gemma-4e"
+model_commit = "3a207ada2c2cd95e9671942e84cf47ea58f0f6af"
+
+seed = 12345
+print_debug_information = true
+
+batch_size = 2
+max_response_length = 10
+kl_divergence_target = 0
+n_trials = 2
+n_startup_trials = 1
+
+export_strategy = "merge"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[good_evaluation_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[bad_evaluation_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"