From 969b48bf6a998e9a84b2a7d03804686c4f5b1c7d Mon Sep 17 00:00:00 2001 From: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com> Date: Tue, 23 Jan 2024 10:47:41 -0500 Subject: [PATCH] Don't use `get_task_dict()` in task registration / initialization (#1331) * don't use get_task_dict() as a helper, it will download the dataset! * pre-commit * Update README.md --------- Co-authored-by: lintangsutawika --- README.md | 2 +- lm_eval/tasks/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8b0433b81d..8342ac317e 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ Note that for externally hosted models, configs such as `--device` and `--batch_ | Your local inference server! | :heavy_check_mark: | `local-completions` or `local-chat-completions` (using `openai-chat-completions` model type) | Any server address that accepts GET requests using HF models and mirror's OpenAI's ChatCompletions interface | `generate_until` | | ... | | `local-completions` (using `openai-completions` model type) | Any server address that accepts GET requests using HF models and mirror's OpenAI's Completions interface | `generate_until` | | ... | -Models which do not supply logits or logprobs can be used with tasks of type `generate_until` only, while models that are local or APIs that supply logprobs/logits can be run on all task types: `generate_until`, `loglikelihood`, `loglikelihood_rolling`, and `multiple_choice`. +Models which do not supply logits or logprobs can be used with tasks of type `generate_until` only, while local models, or APIs that supply logprobs/logits of their prompts, can be run on all task types: `generate_until`, `loglikelihood`, `loglikelihood_rolling`, and `multiple_choice`. For more information on the different task `output_types` and model request types, see [our documentation](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/model_guide.md#interface). diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py index 3eb86f2d05..5b7142496e 100644 --- a/lm_eval/tasks/__init__.py +++ b/lm_eval/tasks/__init__.py @@ -67,12 +67,12 @@ def register_configurable_group(config: Dict[str, str], yaml_path: str = None) - if "task" in task_config: task_name = task_config["task"] if task_name in ALL_TASKS: - task_obj = get_task_dict(task_name)[task_name] + task_obj = TASK_REGISTRY[task_name] if type(task_obj) == tuple: _, task_obj = task_obj if task_obj is not None: - base_config = task_obj._config.to_dict(keep_callable=True) + base_config = task_obj.CONFIG.to_dict(keep_callable=True) task_name_config["task"] = f"{group}_{task_name}" task_config = utils.load_yaml_config(yaml_path, task_config)