From 77770ec79883343d32051cfb6a04f64523cd8df1 Mon Sep 17 00:00:00 2001 From: Yanming Wang Date: Wed, 6 Oct 2021 04:54:54 -0700 Subject: [PATCH] Fix trainer logging_nan_inf_filter in torch_xla mode (#13896) * Fix logging_nan_inf_filter in torch_xla mode * Update src/transformers/trainer.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Fix format Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/trainer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index c44960564cdd4..981007f5a6682 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1311,9 +1311,10 @@ def train( else: tr_loss_step = self.training_step(model, inputs) - if args.logging_nan_inf_filter and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step)): - # if loss is nan or inf simply add the average of previous logged losses - tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged) + if args.logging_nan_inf_filter and not is_torch_tpu_available(): + if torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step): + # if loss is nan or inf simply add the average of previous logged losses + tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged) else: tr_loss += tr_loss_step