From af5a22bbacb407219ca709493bdfe8e2ef9370a4 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 18 Nov 2015 18:06:44 +0100 Subject: [PATCH] Add more connection options for S3 (fixes #8) --- CHANGELOG.txt | 1 + docs/index.rst | 8 ++++++ pyramid_storage/s3.py | 50 +++++++++++++++++++++++++++++------ tests/test_s3.py | 61 +++++++++++++++++++++++++++++++++++++++++-- tox.ini | 1 + 5 files changed, 111 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3c266ad..3ffad10 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -4,6 +4,7 @@ - Bucket name is now read from ``storage.aws.bucket_name`` setting, as stated in documentation. - ACL is now read from ``storage.aws.acl`` setting, as stated in documentation. +- Added new connection options for S3 (fixes #8) 0.0.8 (2014-10-1) ================== diff --git a/docs/index.rst b/docs/index.rst index 7800556..51ca1a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,6 +71,14 @@ Setting Default Description **base_url** Relative or absolute base URL for uploads; must end in slash ("/") **extensions** ``default`` List of extensions or extension groups (see below) **name** ``storage`` Name of property added to request, e.g. **request.storage** + +**use_path_style** ``False`` Use paths for buckets instead of subdomains (useful for testing) +**is_secure** ``True`` Use ``https`` +**host** ``None`` Host for Amazon S3 server (eg. `localhost`) +**port** ``None`` Port for Amazon S3 server (eg. `5000`) +**region** ``None`` Region identifier, *host* and *port* will be ignored +**num_retries** ``1`` Number of retry for connection errors +**timeout** ``5`` HTTP socket timeout in seconds =================== ================= ================================================================== diff --git a/pyramid_storage/s3.py b/pyramid_storage/s3.py index 93f6428..fe65d23 100644 --- a/pyramid_storage/s3.py +++ b/pyramid_storage/s3.py @@ -4,6 +4,7 @@ import mimetypes from pyramid import compat +from pyramid.settings import asbool from zope.interface import implementer from . import utils @@ -28,32 +29,65 @@ class S3FileStorage(object): @classmethod def from_settings(cls, settings, prefix): options = ( - ('aws.access_key', True, None), - ('aws.secret_key', True, ''), ('aws.bucket_name', True, None), ('aws.acl', False, 'public-read'), ('base_url', False, ''), ('extensions', False, 'default'), + # S3 Connection options. + ('aws.access_key', False, None), + ('aws.secret_key', False, None), + ('aws.use_path_style', False, False), + ('aws.is_secure', False, True), + ('aws.host', False, None), + ('aws.port', False, None), + ('aws.region', False, None), + ('aws.num_retries', False, 1), + ('aws.timeout', False, 5), ) kwargs = utils.read_settings(settings, options, prefix) kwargs = dict([(k.replace('aws.', ''), v) for k, v in kwargs.items()]) + kwargs['aws_access_key_id'] = kwargs.pop('access_key') + kwargs['aws_secret_access_key'] = kwargs.pop('secret_key') return cls(**kwargs) - def __init__(self, access_key, secret_key, bucket_name, - acl=None, base_url='', extensions='default'): - self.access_key = access_key - self.secret_key = secret_key + def __init__(self, bucket_name, acl=None, base_url='', + extensions='default', **conn_options): self.bucket_name = bucket_name self.acl = acl self.base_url = base_url self.extensions = resolve_extensions(extensions) + self.conn_options = conn_options def get_connection(self): try: - from boto.s3.connection import S3Connection + import boto except ImportError: raise RuntimeError("You must have boto installed to use s3") - return S3Connection(self.access_key, self.secret_key) + + from boto.s3.connection import OrdinaryCallingFormat + from boto.s3 import connect_to_region + + options = self.conn_options.copy() + options['is_secure'] = asbool(options['is_secure']) + options['port'] = int(options['port']) + + if asbool(options.pop('use_path_style')): + options['calling_format'] = OrdinaryCallingFormat() + + num_retries = int(options.pop('num_retries')) + timeout = float(options.pop('timeout')) + + region = options.pop('region') + if region: + del options['host'] + del options['port'] + conn = connect_to_region(region, **options) + else: + conn = boto.connect_s3(**options) + + conn.num_retries = num_retries + conn.http_connection_kwargs['timeout'] = timeout + return conn def get_bucket(self): return self.get_connection().get_bucket(self.bucket_name) diff --git a/tests/test_s3.py b/tests/test_s3.py index 9208492..121db1d 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -240,10 +240,10 @@ def test_from_settings_with_defaults(): } inst = s3.S3FileStorage.from_settings(settings, 'storage.') assert inst.base_url == '' - assert inst.access_key == 'abc' - assert inst.secret_key == '123' assert inst.bucket_name == 'Attachments' assert inst.acl == 'public-read' + assert inst.conn_options['aws_access_key_id'] == 'abc' + assert inst.conn_options['aws_secret_access_key'] == '123' assert set(('jpg', 'txt', 'doc')).intersection(inst.extensions) @@ -253,3 +253,60 @@ def test_from_settings_if_base_path_missing(): with pytest.raises(pyramid_exceptions.ConfigurationError): s3.S3FileStorage.from_settings({}, 'storage.') + + +def test_from_settings_with_additional_options(): + + from pyramid_storage import s3 + + settings = { + 'storage.aws.access_key': 'abc', + 'storage.aws.secret_key': '123', + 'storage.aws.bucket_name': 'Attachments', + 'storage.aws.is_secure': 'false', + 'storage.aws.host': 'localhost', + 'storage.aws.port': '5000', + 'storage.aws.use_path_style': 'true', + 'storage.aws.num_retries': '3', + 'storage.aws.timeout': '10', + } + inst = s3.S3FileStorage.from_settings(settings, 'storage.') + with mock.patch('boto.connect_s3') as boto_mocked: + boto_mocked.return_value.http_connection_kwargs = {} + conn = inst.get_connection() + assert conn.num_retries == 3 + assert conn.http_connection_kwargs['timeout'] == 10 + + _, boto_options = boto_mocked.call_args_list[0] + + calling_format = boto_options.pop('calling_format') + assert calling_format.__class__.__name__ == 'OrdinaryCallingFormat' + + assert boto_options == { + 'is_secure': False, + 'host': 'localhost', + 'port': 5000, + 'aws_access_key_id': 'abc', + 'aws_secret_access_key': '123' + } + + +def test_from_settings_with_regional_options_ignores_host_port(): + + from pyramid_storage import s3 + + settings = { + 'storage.aws.access_key': 'abc', + 'storage.aws.secret_key': '123', + 'storage.aws.bucket_name': 'Attachments', + 'storage.aws.region': 'eu-west-1', + 'storage.aws.host': 'localhost', + 'storage.aws.port': '5000', + } + inst = s3.S3FileStorage.from_settings(settings, 'storage.') + with mock.patch('boto.s3.connect_to_region') as boto_mocked: + boto_mocked.return_value.http_connection_kwargs = {} + inst.get_connection() + _, boto_options = boto_mocked.call_args_list[0] + assert 'host' not in boto_options + assert 'port' not in boto_options diff --git a/tox.ini b/tox.ini index 4d3c20f..3787492 100644 --- a/tox.ini +++ b/tox.ini @@ -4,4 +4,5 @@ envlist = py26,py27,py34 deps= pytest mock + moto commands=py.test # or 'nosetests' or ...