setting trust_remote_code (EleutherAI#1467)

mycoalchen · Feb 26, 2024 · c1145df · c1145df
1 parent d27c0c0
commit c1145df
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py
@@ -96,7 +96,7 @@ def __init__(
         dtype: Optional[Union[str, torch.dtype]] = "auto",
         batch_size: Optional[Union[int, str]] = 1,
         max_batch_size: Optional[int] = 64,
-        trust_remote_code: Optional[bool] = False,
+        trust_remote_code: Optional[bool] = True,
         use_fast_tokenizer: Optional[bool] = True,
         # arguments used for splitting a model across GPUs naively.
         # only used if `parallelize=True`.

diff --git a/lm_eval/tasks/asdiv/default.yaml b/lm_eval/tasks/asdiv/default.yaml
@@ -12,3 +12,5 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 1.0
+dataset_kwargs:
+  trust_remote_code: true