Add S3 resolve_content

pysparkling/fileio/fs/s3.py

@@ -5,7 +5,7 @@
 import logging
 
 from ...exceptions import FileSystemNotSupported
-from ...utils import Tokenizer
+from ...utils import Tokenizer, parse_file_uri
 from .file_system import FileSystem
 
 log = logging.getLogger(__name__)
@@ -50,6 +50,8 @@ def __init__(self, file_name):
     @classmethod
     def _get_conn(cls):
         if not cls._conn:
+            if boto is None:
+                raise FileSystemNotSupported('S3 not supported. Install "boto".')
             cls._conn = boto.connect_s3(**cls.connection_kwargs)
         return cls._conn
 
@@ -76,6 +78,30 @@ def resolve_filenames(cls, expr):
                 ))
         return files
 
+    @classmethod
+    def resolve_content(cls, expr):
+        scheme, bucket_name, folder_path, pattern = parse_file_uri(expr)
+
+        folder_path = folder_path[1:]  # Remove leading slash
+
+        expr = "{0}{1}".format(folder_path, pattern)
+        # Match all files inside folders that match expr
+        pattern_expr = "{0}{1}*".format(expr, "" if expr.endswith("/") else "/")
+
+        bucket = cls._get_conn().get_bucket(
+            bucket_name,
+            validate=False
+        )
+        files = []
+        for k in bucket.list(prefix=folder_path):
+            if fnmatch(k.name, expr) or fnmatch(k.name, pattern_expr):
+                files.append('{0}://{1}/{2}'.format(
+                    scheme,
+                    bucket_name,
+                    k.name,
+                ))
+        return files
+
     def exists(self):
         t = Tokenizer(self.file_name)
         t.next('//')  # skip scheme