[Feat] Support dataset_suffix check for mixed configs

configs/datasets/GaokaoBench/GaokaoBench_mixed.py

@@ -1,4 +1,4 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .GaokaoBench_mixed_f2038e import GaokaoBench_datasets  # noqa: F401, F403
+    from .GaokaoBench_mixed_9af5ee import GaokaoBench_datasets  # noqa: F401, F403

configs/datasets/agieval/agieval_mixed_0fa998.py

@@ -75,6 +75,13 @@
         _hint = '答案是：'
     else:
         _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template={
+                label: dict(round=[
+                    dict(role='HUMAN', prompt='{question}\n{options}'),
+                    dict(role='BOT', prompt=f'{_hint}{label}')
                 ])
                 for label in _options
             }),

configs/datasets/collections/base_medium.py

@@ -3,8 +3,8 @@
 with read_base():
     from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
     from ..ceval.ceval_ppl_578f8d import ceval_datasets
-    from ..agieval.agieval_mixed_2f14ad import agieval_datasets
-    from ..GaokaoBench.GaokaoBench_mixed_f2038e import GaokaoBench_datasets
+    from ..agieval.agieval_mixed_713d14 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
     from ..bbh.bbh_gen_5b92b0 import bbh_datasets
     from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
     from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets

configs/datasets/collections/base_medium_llama.py

@@ -3,8 +3,8 @@
 with read_base():
     from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
     from ..ceval.ceval_ppl_578f8d import ceval_datasets
-    from ..agieval.agieval_mixed_2f14ad import agieval_datasets
-    from ..GaokaoBench.GaokaoBench_mixed_f2038e import GaokaoBench_datasets
+    from ..agieval.agieval_mixed_713d14 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
     from ..bbh.bbh_gen_5b92b0 import bbh_datasets
     from ..humaneval.humaneval_gen_a82cae import humaneval_datasets
     from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets

configs/datasets/collections/leaderboard/qwen.py

@@ -2,10 +2,10 @@
 
 with read_base():
     from ...ceval.ceval_ppl_578f8d import ceval_datasets
-    from ...agieval.agieval_mixed_2f14ad import agieval_datasets
+    from ...agieval.agieval_mixed_713d14 import agieval_datasets
     from ...mmlu.mmlu_ppl_ac766d import mmlu_datasets
     from ...cmmlu.cmmlu_ppl_8b9c76 import cmmlu_datasets
-    from ...GaokaoBench.GaokaoBench_mixed_f2038e import GaokaoBench_datasets
+    from ...GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
     from ...ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets
     from ...ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
 

configs/datasets/collections/leaderboard/qwen_chat.py

@@ -2,7 +2,7 @@
 
 with read_base():
     from ...ceval.ceval_gen_5f30c7 import ceval_datasets
-    from ...agieval.agieval_mixed_2f14ad import agieval_datasets
+    from ...agieval.agieval_mixed_713d14 import agieval_datasets
     from ...mmlu.mmlu_gen_4d595a import mmlu_datasets
     from ...cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
     from ...GaokaoBench.GaokaoBench_gen_5cfe9e import GaokaoBench_datasets

configs/eval_hf_llama2.py

@@ -6,7 +6,7 @@
     from .datasets.nq.nq_open_gen_e93f8a import nq_datasets
     from .datasets.gsm8k.gsm8k_gen_3309bd import gsm8k_datasets
     from .datasets.humaneval.humaneval_gen_a82cae import humaneval_datasets
-    from .datasets.agieval.agieval_mixed_2f14ad import agieval_datasets
+    from .datasets.agieval.agieval_mixed_713d14 import agieval_datasets
     from .datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314797 import BoolQ_datasets
     from .datasets.hellaswag.hellaswag_ppl_a6e128 import hellaswag_datasets
     from .datasets.obqa.obqa_ppl_6aac9e import obqa_datasets

configs/eval_internlm2_keyset.py

@@ -2,7 +2,7 @@
 
 with read_base():
     from .datasets.mmlu.mmlu_ppl_ac766d import mmlu_datasets
-    from .datasets.agieval.agieval_mixed_2f14ad import agieval_datasets
+    from .datasets.agieval.agieval_mixed_713d14 import agieval_datasets
     from .datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
     from .datasets.gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
     from .datasets.math.math_gen_265cce import math_datasets

tools/update_dataset_suffix.py

@@ -75,7 +75,7 @@ def get_hash(path):
 
 def check_and_rename(filepath):
     base_name = os.path.basename(filepath)
-    match = re.match(r'(.*)_(gen|ppl|ll)_(.*).py', base_name)
+    match = re.match(r'(.*)_(gen|ppl|ll|mixed)_(.*).py', base_name)
     if match:
         dataset, mode, old_hash = match.groups()
         new_hash = get_hash(filepath)