[Refactor] Move fix_id_list to Retriever

configs/datasets/GLUE_CoLA/GULE_CoLA_ppl_77d0df.py

@@ -23,8 +23,8 @@
         },
         ice_token='</E>',
     ),
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=PPLInferencer, fix_id_list=[17, 18, 19, 20, 21]))
+    retriever=dict(type=FixKRetriever, fix_id_list=[17, 18, 19, 20, 21]),
+    inferencer=dict(type=PPLInferencer))
 
 CoLA_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 

configs/datasets/GLUE_QQP/GLUE_QQP_ppl_250d00.py

@@ -22,8 +22,8 @@
         },
         ice_token='</E>',
     ),
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]))
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=PPLInferencer))
 
 QQP_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 

configs/datasets/ceval/ceval_gen_2daf24.py

@@ -161,8 +161,8 @@
                     ]),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=GenInferencer),
         )
 
         ceval_eval_cfg = dict(

configs/datasets/ceval/ceval_gen_5f30c7.py

@@ -161,8 +161,8 @@
                     ]),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=GenInferencer),
         )
 
         ceval_eval_cfg = dict(

configs/datasets/ceval/ceval_ppl_578f8d.py

@@ -163,8 +163,8 @@
                 },
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=PPLInferencer),
         )
 
         ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

configs/datasets/ceval/ceval_ppl_93e5ce.py

@@ -163,8 +163,8 @@
                 },
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=PPLInferencer),
         )
 
         ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

configs/datasets/cmb/cmb_gen_72cbb7.py

@@ -28,8 +28,8 @@
         ),
         ice_token="</E>",
     ),
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=GenInferencer),
 )
 
 cmb_datasets.append(

configs/datasets/cmmlu/cmmlu_gen_c13365.py

@@ -96,8 +96,8 @@
                 ]),
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     cmmlu_eval_cfg = dict(

configs/datasets/cmmlu/cmmlu_ppl_8b9c76.py

@@ -98,8 +98,8 @@
             },
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=PPLInferencer),
     )
 
     cmmlu_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

configs/datasets/mmlu/mmlu_gen_23a9a9.py

@@ -29,8 +29,8 @@
             dict(role='BOT', prompt='{target}\n')
         ])),
     prompt_template=mmlu_prompt_template,
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]))
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=GenInferencer))
 
 mmlu_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),

configs/datasets/mmlu/mmlu_gen_5d1409.py

@@ -102,8 +102,8 @@
             ),
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     mmlu_eval_cfg = dict(

configs/datasets/mmlu/mmlu_gen_79e572.py

@@ -87,8 +87,8 @@
             f"{_hint}</E>{{input}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer:",
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     mmlu_eval_cfg = dict(

configs/datasets/mmlu/mmlu_gen_a484b3.py

@@ -102,8 +102,8 @@
             ),
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     mmlu_eval_cfg = dict(

configs/datasets/mmlu/mmlu_ppl_ac766d.py

@@ -93,8 +93,8 @@
             },
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=PPLInferencer),
     )
 
     mmlu_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )

configs/datasets/nq/nq_gen_0356ec.py

@@ -44,8 +44,8 @@
                 ),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, max_out_len=50, fix_id_list=list(range(k))),
+            retriever=dict(type=FixKRetriever, fix_id_list=list(range(k))),
+            inferencer=dict(type=GenInferencer, max_out_len=50),
         )
 
     nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role="BOT")

configs/datasets/triviaqa/triviaqa_gen_0356ec.py

@@ -45,8 +45,8 @@
                 ),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, max_out_len=50, fix_id_list=list(range(k))),
+            retriever=dict(type=FixKRetriever, fix_id_list=list(range(k))),
+            inferencer=dict(type=GenInferencer, max_out_len=50),
         )
 
     triviaqa_eval_cfg = dict(evaluator=dict(type=TriviaQAEvaluator), pred_role="BOT")

docs/en/prompt/prompt_template.md

@@ -34,8 +34,8 @@ infer_cfg = dict(
         template='Solve the following questions.\n</E>{question}\n{answer}',
         ice_token="</E>"
     ),
-    retriever=dict(type=FixKRetriever),  # Definition of how to retrieve in-context examples.
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1]),  # Method used to generate predictions.
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1]),  # Definition of how to retrieve in-context examples.
+    inferencer=dict(type=GenInferencer),  # Method used to generate predictions.
 )
 ```
 

docs/zh_cn/prompt/prompt_template.md

@@ -34,8 +34,8 @@ infer_cfg=dict(
         template='Solve the following questions.\n</E>{question}\n{answer}',
         ice_token="</E>"
     ),
-    retriever=dict(type=FixKRetriever),  # 定义 in context example 的获取方式
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1]),  # 使用何种方式推理得到 prediction
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1]),  # 定义 in context example 的获取方式
+    inferencer=dict(type=GenInferencer),  # 使用何种方式推理得到 prediction
 )
 ```
 

opencompass/openicl/icl_inferencer/icl_agent_inferencer.py

@@ -55,10 +55,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # Create tmp json file for saving intermediate results and future
         # resuming

opencompass/openicl/icl_inferencer/icl_attack_inferencer.py

@@ -78,7 +78,11 @@ def __init__(
         self.output_column = dataset_cfg['reader_cfg']['output_column']
         self.gen_field_replace_token = gen_field_replace_token
         self.max_out_len = max_out_len
-        self.fix_id_list = fix_id_list
+
+        if fix_id_list:
+            raise ValueError('Passing fix_id_list to Inferencer is no longer '
+                             'allowed. Please pass it to FixKRetriever '
+                             'instead.')
 
         if self.model.is_api and save_every is None:
             save_every = 1
@@ -94,10 +98,7 @@ def predict(self, adv_prompt) -> List:
         output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in self.retriever.__class__.__name__:
-            ice_idx_list = self.retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = self.retriever.retrieve()
+        ice_idx_list = self.retriever.retrieve()
 
         # 3. Generate prompts for testing input
         prompt_list, label_list = self.get_generation_prompt_list_from_retriever_indices(  # noqa

opencompass/openicl/icl_inferencer/icl_clp_inferencer.py

@@ -66,8 +66,11 @@ def __init__(
             **kwargs,
         )
 
-        self.fix_id_list = fix_id_list
         # TODO: support multiple token
+        if fix_id_list:
+            raise ValueError('Passing fix_id_list to Inferencer is no longer '
+                             'allowed. Please pass it to FixKRetriever '
+                             'instead.')
         assert single_token, 'Only support single token choice currently.'
         self.single_token = single_token
 
@@ -103,10 +106,7 @@ def inference(self,
             raise ValueError(err_msg)
 
         # 2. Get results of retrieval process
-        if self.fix_id_list:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate in-context examples for testing inputs
         for idx in range(len(ice_idx_list)):

opencompass/openicl/icl_inferencer/icl_gen_inferencer.py

@@ -64,7 +64,11 @@ def __init__(
 
         self.gen_field_replace_token = gen_field_replace_token
         self.max_out_len = max_out_len
-        self.fix_id_list = fix_id_list
+
+        if fix_id_list:
+            raise ValueError('Passing fix_id_list to Inferencer is no longer '
+                             'allowed. Please pass it to FixKRetriever '
+                             'instead.')
 
         if self.model.is_api and save_every is None:
             save_every = 1
@@ -85,10 +89,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate prompts for testing input
         prompt_list = self.get_generation_prompt_list_from_retriever_indices(
@@ -220,10 +221,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate prompts for testing input
         prompt_list = self.get_generation_prompt_list_from_retriever_indices(

opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py

@@ -53,7 +53,11 @@ def __init__(
         )
 
         self.labels = labels
-        self.fix_id_list = fix_id_list
+
+        if fix_id_list:
+            raise ValueError('Passing fix_id_list to Inferencer is no longer '
+                             'allowed. Please pass it to FixKRetriever '
+                             'instead.')
 
     def inference(self,
                   retriever: BaseRetriever,
@@ -75,10 +79,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if self.fix_id_list:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Get labels of all the classes
         if self.labels is None:

opencompass/openicl/icl_inferencer/icl_sc_inferencer.py

@@ -69,9 +69,13 @@ def __init__(
         self.gen_field_replace_token = gen_field_replace_token
         self.generation_kwargs = generation_kwargs
         self.max_out_len = max_out_len
-        self.fix_id_list = fix_id_list
         self.sc_size = sc_size
 
+        if fix_id_list:
+            raise ValueError('Passing fix_id_list to Inferencer is no longer '
+                             'allowed. Please pass it to FixKRetriever '
+                             'instead.')
+
         if self.model.is_api and save_every is None:
             save_every = 1
         self.save_every = save_every
@@ -91,10 +95,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate prompts for testing input
         prompt_list = self.get_generation_prompt_list_from_retriever_indices(

opencompass/openicl/icl_inferencer/icl_tot_inferencer.py

@@ -319,10 +319,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate prompts for testing input
         prompt_list = self.get_generation_prompt_list_from_retriever_indices(

opencompass/openicl/icl_retriever/icl_fix_k_retriever.py

@@ -19,6 +19,8 @@ class FixKRetriever(BaseRetriever):
     Args:
         dataset (`BaseDataset`): Any BaseDataset instances.
             Attributes of ``reader``, ``train`` and ``test`` will be used.
+        fix_id_list (List[int]): List of in-context example indices for every
+            test prompts.
         ice_separator (`Optional[str]`): The separator between each in-context
             example template when origin `PromptTemplate` is provided. Defaults
             to '\n'.
@@ -31,22 +33,19 @@ class FixKRetriever(BaseRetriever):
 
     def __init__(self,
                  dataset,
+                 fix_id_list: List[int],
                  ice_separator: Optional[str] = '\n',
                  ice_eos_token: Optional[str] = '\n',
                  ice_num: Optional[int] = 1) -> None:
         super().__init__(dataset, ice_separator, ice_eos_token, ice_num)
+        self.fix_id_list = fix_id_list
 
-    def retrieve(self, id_list: List[int]):
-        """Retrieve the in-context example index for each test example.
-
-        Args:
-            id_list (List[int]): List of in-context example indices for every
-                test prompts.
-        """
+    def retrieve(self):
+        """Retrieve the in-context example index for each test example."""
         num_idx = len(self.index_ds)
-        for idx in id_list:
+        for idx in self.fix_id_list:
             assert idx < num_idx, f'Index {idx} is out of range of {num_idx}'
         rtr_idx_list = []
         for _ in trange(len(self.test_ds), disable=not self.is_main_process):
-            rtr_idx_list.append(id_list)
+            rtr_idx_list.append(self.fix_id_list)
         return rtr_idx_list