Add NCCL_PROTO=simple environment variable to handle the out-of-order…

… data delivery from EFA (#196)

Co-authored-by: prasadru <frodo@ip-10-0-12-252.us-west-2.compute.internal>

src/sagemaker_training/torch_distributed.py

@@ -75,7 +75,9 @@ def _setup(self):
         if self._instance_type in SM_EFA_RDMA_INSTANCES:
             # Use EFA's RDMA functionality for one-sided and two-sided transfer
             os.environ["FI_EFA_USE_DEVICE_RDMA"] = "1"
+            os.environ["RDMAV_FORK_SAFE"] = "1"
         os.environ["NCCL_SOCKET_IFNAME"] = str(self._network_interface_name)
+        os.environ["NCCL_PROTO"] = "simple"
 
     def _create_command(self):
         """