add vllm get_ppl

opencompass/models/vllm.py

@@ -1,5 +1,6 @@
 from typing import Dict, List, Optional
-
+import torch
+import numpy as np
 from opencompass.models.base import BaseModel
 from opencompass.utils import get_logger
 
@@ -96,6 +97,25 @@ def generate(self, inputs: List[str], max_out_len: int,
             output_strs.append(generated_text)
 
         return output_strs
+
+    def get_ppl(self,
+                inputs: List[str],
+                mask_length: Optional[List[int]] = None) -> List[float]:
+
+        bsz = len(inputs)
+
+        sampling_kwargs = SamplingParams(prompt_logprobs=0,**self.generation_kwargs)
+        # forward
+        outputs = self.model.generate(inputs, sampling_kwargs)
+        # compute ppl
+        ce_loss = []
+        for i in range(bsz):
+            outputs_prob = outputs[i].prompt_logprobs[1:]
+            prompt_token_ids = outputs[i].prompt_token_ids[1:]
+            outputs_prob_list = [outputs_prob[i][prompt_token_ids[i]] for i in range(len(outputs_prob))]
+            outputs_prob_list = torch.tensor(outputs_prob_list)
+            ce_loss.append(-1 * outputs_prob_list.sum(-1).cpu().detach().numpy() / len(prompt_token_ids))
+        return np.array(ce_loss)
 
     def prompts_preproccess(self, inputs: List[str]):
         if self.use_fastchat_template: