Add more connection options for S3 (fixes #8)

CHANGELOG.txt

@@ -4,6 +4,7 @@
 - Bucket name is now read from ``storage.aws.bucket_name`` setting, as stated
   in documentation.
 - ACL is now read from ``storage.aws.acl`` setting, as stated in documentation.
+- Added new connection options for S3 (fixes #8)
 
 0.0.8 (2014-10-1)
 ==================

docs/index.rst

@@ -71,6 +71,14 @@ Setting                Default                Description
 **base_url**                                  Relative or absolute base URL for uploads; must end in slash ("/")
 **extensions**         ``default``            List of extensions or extension groups (see below)
 **name**               ``storage``            Name of property added to request, e.g. **request.storage**
+
+**use_path_style**     ``False``              Use paths for buckets instead of subdomains (useful for testing)
+**is_secure**          ``True``               Use ``https``
+**host**               ``None``               Host for Amazon S3 server (eg. `localhost`)
+**port**               ``None``               Port for Amazon S3 server (eg. `5000`)
+**region**             ``None``               Region identifier, *host* and *port* will be ignored
+**num_retries**        ``1``                  Number of retry for connection errors
+**timeout**            ``5``                  HTTP socket timeout in seconds
 ===================    =================      ==================================================================
 
 

pyramid_storage/s3.py

@@ -4,6 +4,7 @@
 import mimetypes
 
 from pyramid import compat
+from pyramid.settings import asbool
 from zope.interface import implementer
 
 from . import utils
@@ -28,32 +29,65 @@ class S3FileStorage(object):
     @classmethod
     def from_settings(cls, settings, prefix):
         options = (
-            ('aws.access_key', True, None),
-            ('aws.secret_key', True, ''),
             ('aws.bucket_name', True, None),
             ('aws.acl', False, 'public-read'),
             ('base_url', False, ''),
             ('extensions', False, 'default'),
+            # S3 Connection options.
+            ('aws.access_key', False, None),
+            ('aws.secret_key', False, None),
+            ('aws.use_path_style', False, False),
+            ('aws.is_secure', False, True),
+            ('aws.host', False, None),
+            ('aws.port', False, None),
+            ('aws.region', False, None),
+            ('aws.num_retries', False, 1),
+            ('aws.timeout', False, 5),
         )
         kwargs = utils.read_settings(settings, options, prefix)
         kwargs = dict([(k.replace('aws.', ''), v) for k, v in kwargs.items()])
+        kwargs['aws_access_key_id'] = kwargs.pop('access_key')
+        kwargs['aws_secret_access_key'] = kwargs.pop('secret_key')
         return cls(**kwargs)
 
-    def __init__(self, access_key, secret_key, bucket_name,
-                 acl=None, base_url='', extensions='default'):
-        self.access_key = access_key
-        self.secret_key = secret_key
+    def __init__(self, bucket_name, acl=None, base_url='',
+                 extensions='default', **conn_options):
         self.bucket_name = bucket_name
         self.acl = acl
         self.base_url = base_url
         self.extensions = resolve_extensions(extensions)
+        self.conn_options = conn_options
 
     def get_connection(self):
         try:
-            from boto.s3.connection import S3Connection
+            import boto
         except ImportError:
             raise RuntimeError("You must have boto installed to use s3")
-        return S3Connection(self.access_key, self.secret_key)
+
+        from boto.s3.connection import OrdinaryCallingFormat
+        from boto.s3 import connect_to_region
+
+        options = self.conn_options.copy()
+        options['is_secure'] = asbool(options['is_secure'])
+        options['port'] = int(options['port'])
+
+        if asbool(options.pop('use_path_style')):
+            options['calling_format'] = OrdinaryCallingFormat()
+
+        num_retries = int(options.pop('num_retries'))
+        timeout = float(options.pop('timeout'))
+
+        region = options.pop('region')
+        if region:
+            del options['host']
+            del options['port']
+            conn = connect_to_region(region, **options)
+        else:
+            conn = boto.connect_s3(**options)
+
+        conn.num_retries = num_retries
+        conn.http_connection_kwargs['timeout'] = timeout
+        return conn
 
     def get_bucket(self):
         return self.get_connection().get_bucket(self.bucket_name)

tests/test_s3.py

@@ -240,10 +240,10 @@ def test_from_settings_with_defaults():
     }
     inst = s3.S3FileStorage.from_settings(settings, 'storage.')
     assert inst.base_url == ''
-    assert inst.access_key == 'abc'
-    assert inst.secret_key == '123'
     assert inst.bucket_name == 'Attachments'
     assert inst.acl == 'public-read'
+    assert inst.conn_options['aws_access_key_id'] == 'abc'
+    assert inst.conn_options['aws_secret_access_key'] == '123'
     assert set(('jpg', 'txt', 'doc')).intersection(inst.extensions)
 
 
@@ -253,3 +253,60 @@ def test_from_settings_if_base_path_missing():
 
     with pytest.raises(pyramid_exceptions.ConfigurationError):
         s3.S3FileStorage.from_settings({}, 'storage.')
+
+
+def test_from_settings_with_additional_options():
+
+    from pyramid_storage import s3
+
+    settings = {
+        'storage.aws.access_key': 'abc',
+        'storage.aws.secret_key': '123',
+        'storage.aws.bucket_name': 'Attachments',
+        'storage.aws.is_secure': 'false',
+        'storage.aws.host': 'localhost',
+        'storage.aws.port': '5000',
+        'storage.aws.use_path_style': 'true',
+        'storage.aws.num_retries': '3',
+        'storage.aws.timeout': '10',
+    }
+    inst = s3.S3FileStorage.from_settings(settings, 'storage.')
+    with mock.patch('boto.connect_s3') as boto_mocked:
+        boto_mocked.return_value.http_connection_kwargs = {}
+        conn = inst.get_connection()
+        assert conn.num_retries == 3
+        assert conn.http_connection_kwargs['timeout'] == 10
+
+        _, boto_options = boto_mocked.call_args_list[0]
+
+        calling_format = boto_options.pop('calling_format')
+        assert calling_format.__class__.__name__ == 'OrdinaryCallingFormat'
+
+        assert boto_options == {
+            'is_secure': False,
+            'host': 'localhost',
+            'port': 5000,
+            'aws_access_key_id': 'abc',
+            'aws_secret_access_key': '123'
+        }
+
+
+def test_from_settings_with_regional_options_ignores_host_port():
+
+    from pyramid_storage import s3
+
+    settings = {
+        'storage.aws.access_key': 'abc',
+        'storage.aws.secret_key': '123',
+        'storage.aws.bucket_name': 'Attachments',
+        'storage.aws.region': 'eu-west-1',
+        'storage.aws.host': 'localhost',
+        'storage.aws.port': '5000',
+    }
+    inst = s3.S3FileStorage.from_settings(settings, 'storage.')
+    with mock.patch('boto.s3.connect_to_region') as boto_mocked:
+        boto_mocked.return_value.http_connection_kwargs = {}
+        inst.get_connection()
+        _, boto_options = boto_mocked.call_args_list[0]
+        assert 'host' not in boto_options
+        assert 'port' not in boto_options

tox.ini

@@ -4,4 +4,5 @@ envlist = py26,py27,py34
 deps=
     pytest
     mock
+    moto
 commands=py.test  # or 'nosetests' or ...