fix: increase worker waiting time for ORTE proc (#178)

* fix: increase worker waiting time for ORTE proc

* remove todo tag for passing pylint chec

* ping sagemaker version to avoid a credential error

src/sagemaker_training/mpi.py

@@ -171,8 +171,9 @@ def _wait_orted_process_to_finish():  # type: () -> None
 
 
 def _orted_process():  # pylint: disable=inconsistent-return-statements
-    """Wait a maximum of 5 minutes for orted process to start."""
-    for _ in range(5 * 60):
+    """Wait a maximum of 20 minutes for orted process to start."""
+    # the wait time here should be set to a dynamic value according to cluster size
+    for _ in range(20 * 60):
         procs = [p for p in psutil.process_iter(attrs=["name"]) if p.info["name"] == "orted"]
         if procs:
             logger.info("Process[es]: %s", procs)

tox.ini

@@ -49,7 +49,7 @@ deps =
     pytest-asyncio
     mock
     awslogs
-    sagemaker[local]
+    sagemaker[local]==2.136.0
     numpy
     flask
     gunicorn