Merge pull request #33 from dask/default-s3

Default S3FileSystem

s3fs/core.py

@@ -95,6 +95,7 @@ class S3FileSystem(object):
     b'Hello, world!'
     """
     _conn = {}
+    _singleton = [None]
     connect_timeout = 5
     read_timeout = 15
 
@@ -116,6 +117,18 @@ def __init__(self, anon=None, key=None, secret=None, token=None, **kwargs):
                 logger.debug('Accredited connection failed, trying anonymous')
                 self.anon = True
         self.s3 = self.connect()
+        S3FileSystem._singleton[0] = self
+
+    @classmethod
+    def current(cls):
+        """ Return the most recently created S3FileSystem
+
+        If no S3FileSystem has been created, then create one
+        """
+        if not cls._singleton[0]:
+            return S3FileSystem()
+        else:
+            return cls._singleton[0]
 
     def connect(self, refresh=False):
         """

s3fs/mapping.py

@@ -2,6 +2,9 @@
 from collections import MutableMapping
 import os
 
+from .core import S3FileSystem
+
+
 class S3Map(MutableMapping):
     """Wrap an S3FileSystem as a mutable wrapping.
 
@@ -10,25 +13,25 @@ class S3Map(MutableMapping):
 
     Parameters
     ----------
-    s3 : S3FileSystem
     root : string
         prefix for all the files (perhaps justa  bucket name
+    s3 : S3FileSystem
     check : bool (=True)
         performs a touch at the location, to check writeability.
 
     Examples
     --------
     >>> s3 = s3fs.S3FileSystem() # doctest: +SKIP
-    >>> mw = MapWrapping(s3, 'mybucket/mapstore/') # doctest: +SKIP
-    >>> mw['loc1'] = b'Hello World' # doctest: +SKIP
-    >>> list(mw.keys()) # doctest: +SKIP
+    >>> d = MapWrapping('mybucket/mapstore/', s3=s3) # doctest: +SKIP
+    >>> d['loc1'] = b'Hello World' # doctest: +SKIP
+    >>> list(d.keys()) # doctest: +SKIP
     ['loc1']
-    >>> mw['loc1'] # doctest: +SKIP
+    >>> d['loc1'] # doctest: +SKIP
     b'Hello World'
     """
 
-    def __init__(self, s3, root, check=False):
-        self.s3 = s3
+    def __init__(self, root, s3=None, check=False):
+        self.s3 = s3 or S3FileSystem.current()
         self.root = root
         if check:
             s3.touch(root+'/a')

s3fs/tests/test_mapping.py

@@ -1,58 +1,63 @@
 from s3fs.tests.test_s3fs import s3, test_bucket_name
-from s3fs.mapping import S3Map
+from s3fs import S3Map, S3FileSystem
 
 root = test_bucket_name+'/mapping'
 
 
 def test_simple(s3):
-    mw = S3Map(s3, root)
-    assert not mw
+    d = S3Map(root, s3)
+    assert not d
 
-    assert list(mw) == list(mw.keys()) == []
-    assert list(mw.values()) == []
-    assert list(mw.items()) == []
+    assert list(d) == list(d.keys()) == []
+    assert list(d.values()) == []
+    assert list(d.items()) == []
+
+
+def test_default_s3filesystem(s3):
+    d = S3Map(root)
+    assert d.s3 is s3
 
 
 def test_with_data(s3):
-    mw = S3Map(s3, root)
-    mw['x'] = b'123'
-    assert list(mw) == list(mw.keys()) == ['x']
-    assert list(mw.values()) == [b'123']
-    assert list(mw.items()) == [('x', b'123')]
-    assert mw['x'] == b'123'
-    assert bool(mw)
+    d = S3Map(root, s3)
+    d['x'] = b'123'
+    assert list(d) == list(d.keys()) == ['x']
+    assert list(d.values()) == [b'123']
+    assert list(d.items()) == [('x', b'123')]
+    assert d['x'] == b'123'
+    assert bool(d)
 
     assert s3.walk(root) == [test_bucket_name+'/mapping/x']
-    mw['x'] = b'000'
-    assert mw['x'] == b'000'
+    d['x'] = b'000'
+    assert d['x'] == b'000'
 
-    mw['y'] = b'456'
-    assert mw['y'] == b'456'
-    assert set(mw) == {'x', 'y'}
+    d['y'] = b'456'
+    assert d['y'] == b'456'
+    assert set(d) == {'x', 'y'}
 
-    mw.clear()
-    assert list(mw) == []
+    d.clear()
+    assert list(d) == []
 
 
 def test_complex_keys(s3):
-    mw = S3Map(s3, root)
-    mw[1] = b'hello'
-    assert mw[1] == b'hello'
-    del mw[1]
+    d = S3Map(root, s3)
+    d[1] = b'hello'
+    assert d[1] == b'hello'
+    del d[1]
 
-    mw[1, 2] = b'world'
-    assert mw[1, 2] == b'world'
-    del mw[1, 2]
+    d[1, 2] = b'world'
+    assert d[1, 2] == b'world'
+    del d[1, 2]
 
-    mw['x', 1, 2] = b'hello world'
-    assert mw['x', 1, 2] == b'hello world'
-    print(list(mw))
+    d['x', 1, 2] = b'hello world'
+    assert d['x', 1, 2] == b'hello world'
+    print(list(d))
 
-    assert ('x', 1, 2) in mw
+    assert ('x', 1, 2) in d
 
 
 def test_pickle(s3):
-    d = S3Map(s3, root)
+    d = S3Map(root, s3)
     d['x'] = b'1'
 
     import pickle

s3fs/tests/test_s3fs.py

@@ -608,3 +608,7 @@ def test_bigger_than_block_read(s3):
             if len(data) == 0:
                 break
     assert b''.join(out) == csv_files['2014-01-01.csv']
+
+
+def test_current(s3):
+    assert S3FileSystem.current() is s3