Merge pull request #37 from dask/ssl

PR: Expose use_ssl option for S3 client

s3fs/core.py

@@ -82,6 +82,9 @@ class S3FileSystem(object):
         If not anonymouns, use this key, if specified
     secret : string (None)
         If not anonymous, use this password, if specified
+    use_ssl : bool (True)
+        Whether to use SSL in connections to S3; may be faster without, but
+        insecure
     kwargs : other parameters for boto3 session
 
     Examples
@@ -99,14 +102,15 @@ class S3FileSystem(object):
     connect_timeout = 5
     read_timeout = 15
 
-    def __init__(self, anon=None, key=None, secret=None, token=None, **kwargs):
+    def __init__(self, anon=None, key=None, secret=None, token=None,
+                 use_ssl=True, **kwargs):
         self.anon = anon
         self.key = key
         self.secret = secret
         self.token = token
         self.kwargs = kwargs
         self.dirs = {}
-        self.no_refresh = False
+        self.use_ssl = use_ssl
         if anon is None:
             try:
                 self.anon = False
@@ -139,9 +143,9 @@ def connect(self, refresh=False):
         refresh : bool (True)
             Whether to use cached filelists, if already read
         """
-        anon, key, secret, kwargs, token = (self.anon, self.key, self.secret,
-                                            self.kwargs, self.token)
-        tok = tokenize(anon, key, secret, kwargs, token)
+        anon, key, secret, kwargs, token, ssl = (self.anon, self.key,
+                            self.secret, self.kwargs, self.token, self.use_ssl)
+        tok = tokenize(anon, key, secret, kwargs, token, ssl)
         if refresh:
             self._conn.pop(tok, None)
         if tok not in self._conn:
@@ -153,12 +157,14 @@ def connect(self, refresh=False):
                 conf = Config(connect_timeout=self.connect_timeout,
                               read_timeout=self.read_timeout,
                               signature_version=UNSIGNED)
-                s3 = boto3.Session(**self.kwargs).client('s3', config=conf)
+                s3 = boto3.Session(**self.kwargs).client('s3', config=conf,
+                                                         use_ssl=ssl)
             else:
                 conf = Config(connect_timeout=self.connect_timeout,
                               read_timeout=self.read_timeout)
                 s3 = boto3.Session(self.key, self.secret, self.token,
-                                   **self.kwargs).client('s3', config=conf)
+                                   **self.kwargs).client('s3', config=conf,
+                                                         use_ssl=ssl)
             self._conn[tok] = s3
         return self._conn[tok]
 
@@ -234,7 +240,7 @@ def _ls(self, path, refresh=False):
             path = path[len('s3://'):]
         path = path.rstrip('/')
         bucket, key = split_path(path)
-        if bucket not in self.dirs or (refresh and not self.no_refresh):
+        if bucket not in self.dirs or refresh:
             if bucket == '':
                 # list of buckets
                 if self.anon:

s3fs/mapping.py

@@ -34,13 +34,17 @@ def __init__(self, root, s3=None, check=False):
         self.s3 = s3 or S3FileSystem.current()
         self.root = root
         if check:
-            s3.touch(root+'/a')
-            s3.rm(root+'/a')
+            self.s3.touch(root+'/a')
+            self.s3.rm(root+'/a')
 
     def clear(self):
         """Remove all keys below root - empties out mapping
         """
-        self.s3.rm(self.root, recursive=True)
+        try:
+            self.s3.rm(self.root, recursive=True)
+        except (IOError, OSError):
+            # ignore non-existance of root
+            pass
 
     def _key_to_str(self, key):
         if isinstance(key, (tuple, list)):

s3fs/tests/test_mapping.py

@@ -11,6 +11,7 @@ def test_simple(s3):
     assert list(d) == list(d.keys()) == []
     assert list(d.values()) == []
     assert list(d.items()) == []
+    d = S3Map(root, s3, check=True)
 
 
 def test_default_s3filesystem(s3):
@@ -56,6 +57,16 @@ def test_complex_keys(s3):
     assert ('x', 1, 2) in d
 
 
+def test_clear_empty(s3):
+    d = S3Map(root, s3)
+    d.clear()
+    assert list(d) == []
+    d[1] = b'1'
+    assert list(d) == ['1']
+    d.clear()
+    assert list(d) == []
+
+
 def test_pickle(s3):
     d = S3Map(root, s3)
     d['x'] = b'1'

s3fs/tests/test_s3fs.py

@@ -80,6 +80,11 @@ def test_simple(s3):
         assert out == data
 
 
+def test_ssl_off():
+    s3 = S3FileSystem(use_ssl=False)
+    assert s3.s3.meta.endpoint_url.startswith('http://')
+
+
 def test_tokenize():
     from s3fs.core import tokenize
     a = (1, 2, 3)