support rwkv5-3b learnboard

configs/datasets/collections/leaderboard/rwkv.py

@@ -0,0 +1,38 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ...ceval.ceval_ppl_578f8d import ceval_datasets
+    # from ...bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ...CLUE_CMRC.CLUE_CMRC_gen_1bd3c8 import CMRC_datasets
+    # from ...CLUE_DRCD.CLUE_DRCD_gen_1bd3c8 import DRCD_datasets
+    # from ...CLUE_afqmc.CLUE_afqmc_ppl_6507d7 import afqmc_datasets
+    # from ...FewCLUE_bustm.FewCLUE_bustm_ppl_e53034 import bustm_datasets
+    # from ...FewCLUE_chid.FewCLUE_chid_ppl_8f2872 import chid_datasets
+    # from ...FewCLUE_cluewsc.FewCLUE_cluewsc_ppl_868415 import cluewsc_datasets
+    # from ...FewCLUE_eprstmt.FewCLUE_eprstmt_ppl_f1e631 import eprstmt_datasets
+    # from ...humaneval.humaneval_gen_8e312c import humaneval_datasets
+    # from ...mbpp.mbpp_gen_1e1056 import mbpp_datasets
+    from ...lambada.lambada_gen_217e11 import lambada_datasets
+    # from ...storycloze.storycloze_ppl_496661 import storycloze_datasets
+    # from ...SuperGLUE_AX_b.SuperGLUE_AX_b_ppl_6db806 import AX_b_datasets
+    # from ...SuperGLUE_AX_g.SuperGLUE_AX_g_ppl_66caf3 import AX_g_datasets
+    # from ...SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314b96 import BoolQ_datasets
+    # from ...SuperGLUE_CB.SuperGLUE_CB_ppl_0143fe import CB_datasets
+    from ...SuperGLUE_COPA.SuperGLUE_COPA_ppl_9f3618 import COPA_datasets
+    # from ...SuperGLUE_MultiRC.SuperGLUE_MultiRC_ppl_ced824 import MultiRC_datasets
+    # from ...SuperGLUE_RTE.SuperGLUE_RTE_ppl_66caf3 import RTE_datasets
+    # from ...SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets
+    # from ...SuperGLUE_WiC.SuperGLUE_WiC_ppl_312de9 import WiC_datasets
+    # from ...SuperGLUE_WSC.SuperGLUE_WSC_ppl_d0f531 import WSC_datasets
+    # from ...race.race_ppl_a138cd import race_datasets
+    # from ...math.math_gen_265cce import math_datasets
+    # from ...gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    # from ...summedits.summedits_ppl_1fbeb6 import summedits_datasets
+    # from ...hellaswag.hellaswag_ppl_47bff9 import hellaswag_datasets
+    from ...piqa.piqa_ppl_1cf9f0 import piqa_datasets
+    from ...winogrande.winogrande_ppl_55a66e import winogrande_datasets
+    # from ...obqa.obqa_ppl_c7c154 import obqa_datasets
+    # from ...nq.nq_gen_c788f6 import nq_datasets
+    # from ...triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

configs/eval_rwkv5_3b.py

@@ -0,0 +1,6 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .models.rwkv.rwkv5_3b import models
+    from .datasets.collections.leaderboard.rwkv import datasets
+    from .summarizers.leaderboard import summarizer

configs/models/rwkv/rwkv5_3b.py

@@ -0,0 +1,24 @@
+from opencompass.models import HuggingFaceCausalLM
+
+models = [
+    dict(
+        type=HuggingFaceCausalLM,
+        abbr='rwkv-5-3b',
+        path="RWKV/rwkv-5-world-3b",
+        tokenizer_path='RWKV/rwkv-5-world-3b',
+        model_kwargs=dict(
+            device_map='auto',
+            trust_remote_code=True,
+        ),
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            trust_remote_code=True,
+            use_fast=False,
+        ),
+        max_out_len=256,
+        max_seq_len=2048,
+        batch_size=1,
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
+]

opencompass/models/huggingface.py

@@ -146,6 +146,11 @@ def _load_tokenizer(self, path: str, tokenizer_path: Optional[str],
             self.tokenizer.bos_token = '<s>'
             self.tokenizer.eos_token = '</s>'
             self.tokenizer.pad_token_id = 0
+        if "RWKV/rwkv" in path or \
+            (tokenizer_path and
+                 'RWKV/rwkv' in tokenizer_path):
+            self.logger.warning('We set new pad_token_id for RWKV model')
+            self.tokenizer.pad_token_id = 0
 
     def _set_model_kwargs_torch_dtype(self, model_kwargs):
         if 'torch_dtype' not in model_kwargs:
@@ -407,7 +412,7 @@ def _get_ppl(self,
         shift_labels = inputs['tokens']['input_ids'][..., 1:].contiguous()
 
         loss_fct = torch.nn.CrossEntropyLoss(
-            reduction='none', ignore_index=self.tokenizer.pad_token_id)
+            reduction='none', ignore_index=self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else 0)
         loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),
                         shift_labels.view(-1)).view(shift_labels.size())
 
@@ -419,7 +424,7 @@ def _get_ppl(self,
             loss = loss * mask
 
         lens = (inputs['tokens']['input_ids'] !=
-                self.tokenizer.pad_token_id).sum(-1).cpu().numpy()
+                (self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else 0)).sum(-1).cpu().numpy()
         if mask_length is not None:
             lens -= np.array(mask_length)
         ce_loss = loss.sum(-1).cpu().detach().numpy() / lens