vh/heretic
1
0
Fork 0
mirror of https://github.com/p-e-w/heretic.git synced 2026-09-26 14:01:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(ara): set batch size on HFLM object

Browse Source
This commit is contained in:
Philipp Emanuel Weidmann
2026-04-01 14:34:21 +05:30
parent f7a456bd0c
commit 3b70fe5dfa
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/heretic/evaluator.py
+5 -2
View File
@@ -98,11 +98,14 @@ class Evaluator:
def get_score(self) -> tuple[tuple[float, float], float, int]: def get_score(self) -> tuple[tuple[float, float], float, int]:
if self.settings.use_piqa: if self.settings.use_piqa:
print(" * Running PIQA benchmark...") print(" * Running PIQA benchmark...")
hflm = HFLM(pretrained=self.model.model, tokenizer=self.model.tokenizer) # ty:ignore[invalid-argument-type] hflm = HFLM(
pretrained=self.model.model, # ty:ignore[invalid-argument-type]
tokenizer=self.model.tokenizer, # ty:ignore[invalid-argument-type]
batch_size="auto",
)
results = lm_eval.simple_evaluate( results = lm_eval.simple_evaluate(
model=hflm, model=hflm,
tasks=["piqa"], tasks=["piqa"],
batch_size="auto",
) )
piqa_acc_norm: float = results["results"]["piqa"]["acc_norm,none"] piqa_acc_norm: float = results["results"]["piqa"]["acc_norm,none"]
print(f" * PIQA acc_norm: [bold]{piqa_acc_norm:.4f}[/]") print(f" * PIQA acc_norm: [bold]{piqa_acc_norm:.4f}[/]")
src/heretic/main.py
+1 -1
View File
@@ -1052,6 +1052,7 @@ def run():
hflm = HFLM( hflm = HFLM(
pretrained=model.model, # ty:ignore[invalid-argument-type] pretrained=model.model, # ty:ignore[invalid-argument-type]
tokenizer=model.tokenizer, # ty:ignore[invalid-argument-type] tokenizer=model.tokenizer, # ty:ignore[invalid-argument-type]
batch_size="auto",
) )
table = Table() table = Table()
@@ -1075,7 +1076,6 @@ def run():
results = lm_eval.simple_evaluate( results = lm_eval.simple_evaluate(
model=hflm, model=hflm,
tasks=[benchmark.task], tasks=[benchmark.task],
batch_size="auto",
) )
return results["results"][benchmark.task] return results["results"][benchmark.task]