open-compass · Leymore · May 14, 2024 · May 14, 2024
diff --git a/.pre-commit-config-zh-cn.yaml b/.pre-commit-config-zh-cn.yaml
@@ -45,19 +45,18 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: check-yaml
       - id: end-of-file-fixer
         exclude: |
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: requirements-txt-fixer
       - id: double-quote-string-fixer
-        exclude: configs/
       - id: check-merge-conflict
       - id: fix-encoding-pragma
         args: ["--remove"]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -45,19 +45,18 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: check-yaml
       - id: end-of-file-fixer
         exclude: |
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: requirements-txt-fixer
       - id: double-quote-string-fixer
-        exclude: configs/
       - id: check-merge-conflict
       - id: fix-encoding-pragma
         args: ["--remove"]

diff --git a/configs/api_examples/eval_api_360.py b/configs/api_examples/eval_api_360.py
@@ -17,7 +17,7 @@
         abbr='360GPT_S2_V9',
         type=AI360GPT,
         path='360GPT_S2_V9',
-        key="xxxxxxxxxxxx",
+        key='xxxxxxxxxxxx',
         generation_kwargs={
             'temperature': 0.9,
             'max_tokens': 2048,
@@ -40,4 +40,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir ="./output/api_360GPT_S2_V9"
+work_dir ='./output/api_360GPT_S2_V9'
diff --git a/configs/api_examples/eval_api_baichuan.py b/configs/api_examples/eval_api_baichuan.py
@@ -18,8 +18,8 @@
         type=BaiChuan,
         path='Baichuan2-53B',
         api_key='xxxxxx',
-        secret_key="xxxxx",
-        url="xxxxx",
+        secret_key='xxxxx',
+        url='xxxxx',
         generation_kwargs={
             'temperature': 0.3,
             'top_p': 0.85,
@@ -41,4 +41,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_baichuan53b/"
+work_dir = 'outputs/api_baichuan53b/'
diff --git a/configs/api_examples/eval_api_baidu.py b/configs/api_examples/eval_api_baidu.py
@@ -39,4 +39,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_erniebot/"
+work_dir = 'outputs/api_erniebot/'
diff --git a/configs/api_examples/eval_api_bytedance.py b/configs/api_examples/eval_api_bytedance.py
@@ -18,8 +18,8 @@
         abbr='skylark-pro-public',
         type=ByteDance,
         path='skylark-pro-public',
-        accesskey="xxxxxxx",
-        secretkey="xxxxxxx",
+        accesskey='xxxxxxx',
+        secretkey='xxxxxxx',
         url='xxxxxx',
         generation_kwargs={
             'temperature': 0.7,
@@ -41,4 +41,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_bytedance/"
+work_dir = 'outputs/api_bytedance/'
diff --git a/configs/api_examples/eval_api_minimax.py b/configs/api_examples/eval_api_minimax.py
@@ -34,4 +34,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_minimax/"
+work_dir = 'outputs/api_minimax/'
diff --git a/configs/api_examples/eval_api_moonshot.py b/configs/api_examples/eval_api_moonshot.py
@@ -37,4 +37,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_moonshot/"
+work_dir = 'outputs/api_moonshot/'
diff --git a/configs/api_examples/eval_api_nanbeige.py b/configs/api_examples/eval_api_nanbeige.py
@@ -18,7 +18,7 @@
         abbr='nanbeige-plus',
         type=Nanbeige,
         path='nanbeige-plus',
-        key="xxxxxx",
+        key='xxxxxx',
         query_per_second=1,
         max_out_len=2048,
         batch_size=8),
@@ -33,4 +33,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir ="./output/nanbeige-plus"
+work_dir ='./output/nanbeige-plus'
diff --git a/configs/api_examples/eval_api_pangu.py b/configs/api_examples/eval_api_pangu.py
@@ -17,13 +17,13 @@
         abbr='pangu',
         type=PanGu,
         path='pangu',
-        access_key="xxxxxx",
-        secret_key="xxxxxx",
-        url = "xxxxxx",
+        access_key='xxxxxx',
+        secret_key='xxxxxx',
+        url = 'xxxxxx',
         # url of token sever, used for generate token, like "https://xxxxxx.myhuaweicloud.com/v3/auth/tokens",
-        token_url = "xxxxxx",
+        token_url = 'xxxxxx',
         # scope-project-name, used for generate token
-        project_name = "xxxxxx",
+        project_name = 'xxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -39,4 +39,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_pangu/"
+work_dir = 'outputs/api_pangu/'
diff --git a/configs/api_examples/eval_api_qwen.py b/configs/api_examples/eval_api_qwen.py
@@ -37,4 +37,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_qwen/"
+work_dir = 'outputs/api_qwen/'
diff --git a/configs/api_examples/eval_api_sensetime.py b/configs/api_examples/eval_api_sensetime.py
@@ -24,17 +24,17 @@
         max_seq_len=2048,
         batch_size=8,
         parameters={
-            "temperature": 0.8,
-            "top_p": 0.7,
-            "max_new_tokens": 1024,
-            "repetition_penalty": 1.05,
-            "know_ids": [],
-            "stream": True,
-            "user": "#*#***TestUser***#*#",
-            "knowledge_config": {
-                "control_level": "normal",
-                "knowledge_base_result": False,
-                "online_search_result": False
+            'temperature': 0.8,
+            'top_p': 0.7,
+            'max_new_tokens': 1024,
+            'repetition_penalty': 1.05,
+            'know_ids': [],
+            'stream': True,
+            'user': '#*#***TestUser***#*#',
+            'knowledge_config': {
+                'control_level': 'normal',
+                'knowledge_base_result': False,
+                'online_search_result': False
             }
         }
     )
@@ -49,4 +49,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_sensetime/"
+work_dir = 'outputs/api_sensetime/'
diff --git a/configs/api_examples/eval_api_xunfei.py b/configs/api_examples/eval_api_xunfei.py
@@ -17,22 +17,22 @@
     dict(
         abbr='Spark-v1-1',
         type=XunFei,
-        appid="xxxx",
+        appid='xxxx',
         path='ws://spark-api.xf-yun.com/v1.1/chat',
-        api_secret = "xxxxxxx",
-        api_key = "xxxxxxx",
+        api_secret = 'xxxxxxx',
+        api_key = 'xxxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
         batch_size=8),
     dict(
         abbr='Spark-v3-1',
         type=XunFei,
-        appid="xxxx",
+        appid='xxxx',
         domain='generalv3',
         path='ws://spark-api.xf-yun.com/v3.1/chat',
-        api_secret = "xxxxxxxx",
-        api_key = "xxxxxxxxx",
+        api_secret = 'xxxxxxxx',
+        api_key = 'xxxxxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -48,4 +48,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_xunfei/"
+work_dir = 'outputs/api_xunfei/'
diff --git a/configs/api_examples/eval_api_zhipu.py b/configs/api_examples/eval_api_zhipu.py
@@ -29,7 +29,7 @@
         abbr='chatglm_pro',
         type=ZhiPuAI,
         path='chatglm_pro',
-        key='xxxxxxxxxxxx', 
+        key='xxxxxxxxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -45,4 +45,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_zhipu/"
+work_dir = 'outputs/api_zhipu/'
diff --git a/configs/api_examples/eval_api_zhipu_v2.py b/configs/api_examples/eval_api_zhipu_v2.py
@@ -64,4 +64,4 @@
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_zhipu_v2/"
+work_dir = 'outputs/api_zhipu_v2/'
diff --git a/configs/dataset_collections/chat_OC15.py b/configs/dataset_collections/chat_OC15.py
@@ -19,4 +19,4 @@
     from ..datasets.gpqa.gpqa_gen_4baadb import gpqa_datasets
     from ..datasets.IFEval.IFEval_gen_3321a3 import ifeval_datasets
 
-datasets = sum((v for k, v in locals().items() if k.endswith("_datasets")), [])
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
diff --git a/configs/datasets/ARC_c/ARC_c_clean_ppl.py b/configs/datasets/ARC_c/ARC_c_clean_ppl.py
@@ -12,29 +12,29 @@
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            "A":
+            'A':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textA}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}')
                 ], ),
-            "B":
+            'B':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textB}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}')
                 ], ),
-            "C":
+            'C':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textC}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}')
                 ], ),
-            "D":
+            'D':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textD}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}')
                 ], ),
         }),
     retriever=dict(type=ZeroRetriever),

diff --git a/configs/datasets/ARC_c/ARC_c_gen_1e0de5.py b/configs/datasets/ARC_c/ARC_c_gen_1e0de5.py
@@ -6,18 +6,18 @@
 from opencompass.utils.text_postprocessors import first_option_postprocess
 
 ARC_c_reader_cfg = dict(
-    input_columns=["question", "textA", "textB", "textC", "textD"],
-    output_column="answerKey")
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
 
 ARC_c_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template=dict(
             round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:"
+                    'Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:'
                 )
             ], ),
     ),
@@ -27,15 +27,15 @@
 
 ARC_c_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
 )
 
 ARC_c_datasets = [
     dict(
-        abbr="ARC-c",
+        abbr='ARC-c',
         type=ARCDataset,
-        path="./data/ARC/ARC-c/ARC-Challenge-Dev.jsonl",
+        path='./data/ARC/ARC-c/ARC-Challenge-Dev.jsonl',
         reader_cfg=ARC_c_reader_cfg,
         infer_cfg=ARC_c_infer_cfg,
         eval_cfg=ARC_c_eval_cfg,

diff --git a/configs/datasets/ARC_c/ARC_c_ppl_2ef631.py b/configs/datasets/ARC_c/ARC_c_ppl_2ef631.py
@@ -14,10 +14,10 @@
         template={
             opt: dict(
                 round=[
-                    dict(role="HUMAN", prompt=f"{{question}}\nA. {{textA}}\nB. {{textB}}\nC. {{textC}}\nD. {{textD}}"),
-                    dict(role="BOT", prompt=f"Answer: {opt}"),
+                    dict(role='HUMAN', prompt=f'{{question}}\nA. {{textA}}\nB. {{textB}}\nC. {{textC}}\nD. {{textD}}'),
+                    dict(role='BOT', prompt=f'Answer: {opt}'),
                 ]
-            ) for opt in ["A", "B", "C", "D"]
+            ) for opt in ['A', 'B', 'C', 'D']
         },
     ),
     retriever=dict(type=ZeroRetriever),