cherrpick bug fix dpop & optrimizer

paddlenlp/trl/dpo_criterion.py

@@ -75,9 +75,10 @@ def dpo_loss(self, policy_chosen_logps, policy_rejected_logps, reference_chosen_
             # for the IPO loss, denoted by tau in the paper.
             loss = (logits - 1 / (2 * self.dpo_config.beta)) ** 2
         elif self.dpo_config.loss_type == "dpop":
-            loss = -F.log_sigmoid(self.dpo_config.beta * logits)
             positive_reg = reference_chosen_logps - policy_chosen_logps
-            loss += self.dpo_config.dpop_lambda * paddle.clip(positive_reg, min=0)
+            loss = -F.log_sigmoid(
+                self.dpo_config.beta * (logits - self.dpo_config.dpop_lambda * paddle.clip(positive_reg, min=0))
+            )
         elif self.dpo_config.loss_type == "kto_pair":
             # eqn (7) of the HALOs paper
             chosen_KL = (policy_chosen_logps - reference_chosen_logps).mean().clip(min=0)

paddlenlp/utils/__init__.py

@@ -22,7 +22,11 @@
 from .infohub import infohub
 from .initializer import to
 from .memory_utils import empty_device_cache
-from .optimizer import *
+
+try:
+    from .optimizer import *
+except:
+    pass
 from .serialization import load_torch
 
 # hack impl for EagerParamBase to function