Revert incorrect fix.

NVIDIA · Jan 29, 2022 · cd49955 · cd49955
1 parent 2a34e0e
commit cd49955
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/megatron/utils.py b/megatron/utils.py
@@ -84,7 +84,7 @@ def average_losses_across_data_parallel_group(losses):
         [loss.clone().detach().view(1) for loss in losses])
     torch.distributed.all_reduce(averaged_losses,
                                  group=mpu.get_data_parallel_group())
-    averaged_losses = averaged_losses /
+    averaged_losses = averaged_losses / \
         torch.distributed.get_world_size(group=mpu.get_data_parallel_group())
 
     return averaged_losses