avoid retrying to download for FileNotFoundError (#409)

Signed-off-by: Jinzhe Zeng <jinzhe.zeng@rutgers.edu>
deepmodeling · Nov 12, 2023 · 8c24fd0 · 8c24fd0
1 parent bd71277
commit 8c24fd0
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 9 deletions.
diff --git a/dpdispatcher/ssh_context.py b/dpdispatcher/ssh_context.py
@@ -940,21 +940,35 @@ def _get_files(self, files, tar_compress=True):
         per_nfile = 100
         ntar = len(files) // per_nfile + 1
         if ntar <= 1:
-            self.block_checkcall(
-                "tar {} {} {}".format(
-                    tar_command,
-                    shlex.quote(of),
-                    " ".join([shlex.quote(file) for file in files]),
+            try:
+                self.block_checkcall(
+                    "tar {} {} {}".format(
+                        tar_command,
+                        shlex.quote(of),
+                        " ".join([shlex.quote(file) for file in files]),
+                    )
                 )
-            )
+            except RuntimeError as e:
+                if "No such file or directory" in str(e):
+                    raise FileNotFoundError(
+                        "Any of the backward files does not exist in the remote directory."
+                    ) from e
+                raise e
         else:
             file_list_file = os.path.join(
                 self.remote_root, ".tmp.tar." + str(uuid.uuid4())
             )
             self.write_file(file_list_file, "\n".join(files))
-            self.block_checkcall(
-                f"tar {tar_command} {shlex.quote(of)} -T {shlex.quote(file_list_file)}"
-            )
+            try:
+                self.block_checkcall(
+                    f"tar {tar_command} {shlex.quote(of)} -T {shlex.quote(file_list_file)}"
+                )
+            except RuntimeError as e:
+                if "No such file or directory" in str(e):
+                    raise FileNotFoundError(
+                        "Any of the backward files does not exist in the remote directory."
+                    ) from e
+                raise e
         # trans
         from_f = pathlib.PurePath(os.path.join(self.remote_root, of)).as_posix()
         to_f = pathlib.PurePath(os.path.join(self.local_root, of)).as_posix()

diff --git a/dpdispatcher/submission.py b/dpdispatcher/submission.py
@@ -274,6 +274,9 @@ def try_download_result(self):
             try:
                 self.download_jobs()
                 success = True
+            except FileNotFoundError as e:
+                # retry will never success if the file is not found
+                raise e
             except (EOFError, Exception) as e:
                 dlog.exception(e)
                 elapsed_time = time.time() - start_time