From 42d5c4bfcf6dc995f3b1f45679372231552ea807 Mon Sep 17 00:00:00 2001 From: LSinev Date: Mon, 3 Jun 2024 16:39:12 +0300 Subject: [PATCH] Fix fewshot seed only set when overriding num_fewshot (#1914) Fix #1906 --- lm_eval/evaluator.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index b70926b6ae..37301718a7 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -241,14 +241,15 @@ def simple_evaluate( f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}" ) task_obj.set_config(key="num_fewshot", value=num_fewshot) - task_obj.set_fewshot_seed(seed=fewshot_random_seed) - eval_logger.info( - f"Setting fewshot random generator seed to {fewshot_random_seed}" - ) else: # if num_fewshot not provided, and the task does not define a default one, default to 0 if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None: task_obj.set_config(key="num_fewshot", value=0) + # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file) + task_obj.set_fewshot_seed(seed=fewshot_random_seed) + eval_logger.info( + f"Setting fewshot random generator seed to {fewshot_random_seed}" + ) if check_integrity: run_task_tests(task_list=tasks)