[Feat] Support dataset_suffix check for mixed configs (#973)

* [Feat] Support dataset_suffix check for mixed configs * update mixed suffix * update suffix --------- Co-authored-by: Leymore <zfz-960727@163.com>
open-compass · May 14, 2024 · 3dbba11 · 3dbba11
1 parent 7505b3c
commit 3dbba11
Show file tree

Hide file tree

Showing 12 changed files with 18 additions and 229 deletions.
diff --git a/configs/datasets/GaokaoBench/GaokaoBench_mixed.py b/configs/datasets/GaokaoBench/GaokaoBench_mixed.py
@@ -1,4 +1,4 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .GaokaoBench_mixed_f2038e import GaokaoBench_datasets  # noqa: F401, F403
+    from .GaokaoBench_mixed_9af5ee import GaokaoBench_datasets  # noqa: F401, F403
diff --git a/...s/GaokaoBench/GaokaoBench_mixed_f2038e.py → ...s/GaokaoBench/GaokaoBench_mixed_9af5ee.py b/...s/GaokaoBench/GaokaoBench_mixed_f2038e.py → ...s/GaokaoBench/GaokaoBench_mixed_9af5ee.py
diff --git a/...mathbench_2024_wocircular_mixed_649349.py → ...mathbench_2024_wocircular_mixed_dc0207.py b/...mathbench_2024_wocircular_mixed_649349.py → ...mathbench_2024_wocircular_mixed_dc0207.py
diff --git a/configs/datasets/agieval/agieval_mixed_0fa998.py b/configs/datasets/agieval/agieval_mixed_0fa998.py
@@ -75,6 +75,13 @@
         _hint = '答案是：'
     else:
         _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template={
+                label: dict(round=[
+                    dict(role='HUMAN', prompt='{question}\n{options}'),
+                    dict(role='BOT', prompt=f'{_hint}{label}')
                 ])
                 for label in _options
             }),

diff --git a/configs/datasets/agieval/agieval_mixed_2f14ad.py b/configs/datasets/agieval/agieval_mixed_2f14ad.py
diff --git a/configs/datasets/collections/base_medium.py b/configs/datasets/collections/base_medium.py
@@ -3,8 +3,8 @@
 with read_base():
     from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
     from ..ceval.ceval_ppl_578f8d import ceval_datasets
-    from ..agieval.agieval_mixed_2f14ad import agieval_datasets
-    from ..GaokaoBench.GaokaoBench_mixed_f2038e import GaokaoBench_datasets
+    from ..agieval.agieval_mixed_713d14 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
     from ..bbh.bbh_gen_5b92b0 import bbh_datasets
     from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
     from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets

diff --git a/configs/datasets/collections/base_medium_llama.py b/configs/datasets/collections/base_medium_llama.py
@@ -3,8 +3,8 @@
 with read_base():
     from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
     from ..ceval.ceval_ppl_578f8d import ceval_datasets
-    from ..agieval.agieval_mixed_2f14ad import agieval_datasets
-    from ..GaokaoBench.GaokaoBench_mixed_f2038e import GaokaoBench_datasets
+    from ..agieval.agieval_mixed_713d14 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
     from ..bbh.bbh_gen_5b92b0 import bbh_datasets
     from ..humaneval.humaneval_gen_a82cae import humaneval_datasets
     from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets

diff --git a/configs/datasets/collections/leaderboard/qwen.py b/configs/datasets/collections/leaderboard/qwen.py
@@ -2,10 +2,10 @@
 
 with read_base():
     from ...ceval.ceval_ppl_578f8d import ceval_datasets
-    from ...agieval.agieval_mixed_2f14ad import agieval_datasets
+    from ...agieval.agieval_mixed_713d14 import agieval_datasets
     from ...mmlu.mmlu_ppl_ac766d import mmlu_datasets
     from ...cmmlu.cmmlu_ppl_8b9c76 import cmmlu_datasets
-    from ...GaokaoBench.GaokaoBench_mixed_f2038e import GaokaoBench_datasets
+    from ...GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
     from ...ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets
     from ...ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
 

diff --git a/configs/datasets/collections/leaderboard/qwen_chat.py b/configs/datasets/collections/leaderboard/qwen_chat.py
@@ -2,7 +2,7 @@
 
 with read_base():
     from ...ceval.ceval_gen_5f30c7 import ceval_datasets
-    from ...agieval.agieval_mixed_2f14ad import agieval_datasets
+    from ...agieval.agieval_mixed_713d14 import agieval_datasets
     from ...mmlu.mmlu_gen_4d595a import mmlu_datasets
     from ...cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
     from ...GaokaoBench.GaokaoBench_gen_5cfe9e import GaokaoBench_datasets

diff --git a/configs/eval_hf_llama2.py b/configs/eval_hf_llama2.py
@@ -6,7 +6,7 @@
     from .datasets.nq.nq_open_gen_e93f8a import nq_datasets
     from .datasets.gsm8k.gsm8k_gen_3309bd import gsm8k_datasets
     from .datasets.humaneval.humaneval_gen_a82cae import humaneval_datasets
-    from .datasets.agieval.agieval_mixed_2f14ad import agieval_datasets
+    from .datasets.agieval.agieval_mixed_713d14 import agieval_datasets
     from .datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314797 import BoolQ_datasets
     from .datasets.hellaswag.hellaswag_ppl_a6e128 import hellaswag_datasets
     from .datasets.obqa.obqa_ppl_6aac9e import obqa_datasets

diff --git a/configs/eval_internlm2_keyset.py b/configs/eval_internlm2_keyset.py
@@ -2,7 +2,7 @@
 
 with read_base():
     from .datasets.mmlu.mmlu_ppl_ac766d import mmlu_datasets
-    from .datasets.agieval.agieval_mixed_2f14ad import agieval_datasets
+    from .datasets.agieval.agieval_mixed_713d14 import agieval_datasets
     from .datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
     from .datasets.gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
     from .datasets.math.math_gen_265cce import math_datasets

diff --git a/tools/update_dataset_suffix.py b/tools/update_dataset_suffix.py
@@ -75,7 +75,7 @@ def get_hash(path):
 
 def check_and_rename(filepath):
     base_name = os.path.basename(filepath)
-    match = re.match(r'(.*)_(gen|ppl|ll)_(.*).py', base_name)
+    match = re.match(r'(.*)_(gen|ppl|ll|mixed)_(.*).py', base_name)
     if match:
         dataset, mode, old_hash = match.groups()
         new_hash = get_hash(filepath)