From 9a7c45e8fb946cafff2e9d9ba270f7c1d3083174 Mon Sep 17 00:00:00 2001 From: Kamal Sharma Date: Thu, 13 Jan 2022 22:41:14 +0530 Subject: [PATCH] Minor formatting and fixes --- horovod/spark/common/store.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/horovod/spark/common/store.py b/horovod/spark/common/store.py index f5b08cd973..c8011c20d7 100644 --- a/horovod/spark/common/store.py +++ b/horovod/spark/common/store.py @@ -316,12 +316,12 @@ def sync_fn(self, run_id): def fn(local_run_path): print(f"Syncing dir {local_run_path} to dir {run_path}") if self.fs.exists(run_path): - local_run_path = os.path.join(local_run_path,self.get_wildcard()) + local_run_path = os.path.join(local_run_path, self.get_wildcard()) self.copy(local_run_path, run_path, recursive=True, overwrite=True) return fn - def copy(self,lpath,rpath,recursive=False,callback=_DEFAULT_CALLBACK,**kwargs): + def copy(self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK,**kwargs): from fsspec.implementations.local import LocalFileSystem, make_path_posix from fsspec.utils import other_paths @@ -334,8 +334,9 @@ def copy(self,lpath,rpath,recursive=False,callback=_DEFAULT_CALLBACK,**kwargs): lpath = make_path_posix(lpath) fs = LocalFileSystem() lpaths = fs.expand_path(lpath, recursive=recursive) + is_wildcard = True if lpath.endswith('/*') else False rpaths = other_paths( - lpaths, rpath, exists=isinstance(rpath, str) and self.fs.isdir(rpath) + lpaths, rpath, exists=isinstance(rpath, str) and self.fs.isdir(rpath) and not is_wildcard ) callback.set_size(len(rpaths))