Add more stats to the training log

espnet · Apr 27, 2023 · c6a1b57 · c6a1b57
1 parent b973527
commit c6a1b57
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/espnet2/train/trainer.py b/espnet2/train/trainer.py
@@ -678,6 +678,17 @@ def train_one_epoch(
                             scaler.update()
 
                 else:
+                    reporter.register(
+                        {
+                            "grad_norm": grad_norm,
+                            "clip": torch.where(
+                                grad_norm > grad_clip,
+                                grad_norm.new_tensor(100),
+                                grad_norm.new_tensor(0),
+                            ),
+                            "loss_scale": scaler.get_scale() if scaler else 1.0,
+                        }
+                    )
                     all_steps_are_invalid = False
                     with reporter.measure_time("optim_step_time"):
                         for iopt, (optimizer, scheduler) in enumerate(