Parallelization of commands (CHIP and PREDICT)

rastervision/filesystem/filesystem.py

@@ -33,7 +33,14 @@ def matches_uri(uri: str, mode: str) -> bool:
 
     @staticmethod
     @abstractmethod
-    def file_exists(uri: str) -> bool:
+    def file_exists(uri: str, include_dir: bool = True) -> bool:
+        """Check if a  file exists.
+        Args:
+          uri: The URI to check
+          include_dir: Include directories in check, if this filesystem
+                       supports directory reads. Otherwise only
+                       return true if a single file exists at the URI.
+        """
         pass  # pragma: no cover
 
     @staticmethod

rastervision/filesystem/http_filesystem.py

@@ -16,7 +16,7 @@ def matches_uri(uri: str, mode: str) -> bool:
         return parsed_uri.scheme in ['http', 'https']
 
     @staticmethod
-    def file_exists(uri: str) -> bool:
+    def file_exists(uri: str, include_dir: bool = True) -> bool:
         try:
             response = urllib.request.urlopen(uri)
             if response.getcode() == 200:

rastervision/filesystem/local_filesystem.py

@@ -39,8 +39,8 @@ def matches_uri(uri: str, mode: str) -> bool:
         return True
 
     @staticmethod
-    def file_exists(uri: str) -> bool:
-        return os.path.isfile(uri)
+    def file_exists(uri: str, include_dir: bool = True) -> bool:
+        return (os.path.isfile(uri) or (include_dir and os.path.isdir(uri)))
 
     @staticmethod
     def read_str(file_uri: str) -> str:

rastervision/filesystem/s3_filesystem.py

@@ -78,19 +78,23 @@ def matches_uri(uri: str, mode: str) -> bool:
         return parsed_uri.scheme == 's3'
 
     @staticmethod
-    def file_exists(uri: str) -> bool:
+    def file_exists(uri: str, include_dir: bool = True) -> bool:
         # Lazily load boto
         import botocore
 
-        s3 = S3FileSystem.get_session().resource('s3')
+        s3 = S3FileSystem.get_session().client('s3')
         parsed_uri = urlparse(uri)
         bucket = parsed_uri.netloc
         key = parsed_uri.path[1:]
+
         try:
-            s3.Object(bucket, key).load()
+            response = s3.list_objects_v2(Bucket=bucket, Prefix=key, MaxKeys=1)
+            if response['KeyCount'] == 0:
+                return False
+            response_key = response['Contents'][0]['Key']
+            return (response_key == key) or include_dir
         except botocore.exceptions.ClientError as e:
             return False
-        return True
 
     @staticmethod
     def read_str(uri: str) -> str:

rastervision/utils/files.py

@@ -156,10 +156,10 @@ def download_or_copy(uri, target_dir, fs=None):
     return local_path
 
 
-def file_exists(uri, fs=None):
+def file_exists(uri, fs=None, include_dir=True):
     if not fs:
         fs = FileSystem.get_file_system(uri, 'r')
-    return fs.file_exists(uri)
+    return fs.file_exists(uri, include_dir)
 
 
 def list_paths(uri, ext='', fs=None):

tests/utils/test_files.py

@@ -337,6 +337,23 @@ def test_list_paths_s3(self):
         list_paths(s3_directory)
         self.assertEqual(len(list_paths(s3_directory)), 1)
 
+    def test_file_exists(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        s3_path = 's3://{}/xxx/lorem.txt'.format(self.bucket_name)
+        s3_directory = 's3://{}/xxx/'.format(self.bucket_name)
+        directory = os.path.dirname(path)
+        make_dir(directory, check_empty=False)
+
+        with open(path, 'w+') as file:
+            file.write(self.lorem)
+        upload_or_copy(path, s3_path)
+
+        self.assertTrue(file_exists(s3_directory, include_dir=True))
+        self.assertTrue(file_exists(s3_path, include_dir=False))
+        self.assertFalse(file_exists(s3_directory, include_dir=False))
+        self.assertFalse(file_exists(s3_directory + 'NOTPOSSIBLE', include_dir=False))
+
+
 
 class TestLocalMisc(unittest.TestCase):
     def setUp(self):
@@ -428,6 +445,21 @@ def test_last_modified(self):
 
         self.assertTrue(isinstance(stamp, datetime.datetime))
 
+    def test_file_exists(self):
+        fs = FileSystem.get_file_system(self.temp_dir.name, 'r')
+
+        path1 = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        dir1 = os.path.dirname(path1)
+        make_dir(dir1, check_empty=False)
+
+        with open(path1, 'w+') as file:
+            file.write(self.lorem)
+
+        self.assertTrue(fs.file_exists(dir1, include_dir=True))
+        self.assertTrue(fs.file_exists(path1, include_dir=False))
+        self.assertFalse(fs.file_exists(dir1, include_dir=False))
+        self.assertFalse(fs.file_exists(dir1 + 'NOTPOSSIBLE', include_dir=False))
+
 
 class TestHttpMisc(unittest.TestCase):
     def setUp(self):