From c9cedd0ab6f87b966bf809035c8d60f57921f646 Mon Sep 17 00:00:00 2001 From: "Javier M. Mellid" Date: Tue, 17 May 2016 09:07:17 +0000 Subject: [PATCH] Add Ceph S3 RGW storage drivers Add Ceph S3 RGW drivers and related stuff in the drivers/rgw.py module: - S3 RGW driver - S3 RGW Outscale driver Signed-off-by: Javier M. Mellid --- libcloud/storage/drivers/rgw.py | 137 ++++++++++++++++++++++++++++++++ libcloud/storage/drivers/s3.py | 80 +------------------ libcloud/storage/providers.py | 4 +- libcloud/storage/types.py | 6 +- 4 files changed, 145 insertions(+), 82 deletions(-) create mode 100644 libcloud/storage/drivers/rgw.py diff --git a/libcloud/storage/drivers/rgw.py b/libcloud/storage/drivers/rgw.py new file mode 100644 index 0000000000..684047f6bd --- /dev/null +++ b/libcloud/storage/drivers/rgw.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcloud.common.types import LibcloudError + +from libcloud.storage.drivers.s3 import BaseS3Connection, S3Connection,\ + S3StorageDriver, API_VERSION + +from libcloud.common.aws import SignedAWSConnection, DEFAULT_SIGNATURE_VERSION + +S3_RGW_DEFAULT_REGION = 'default' + +S3_RGW_OUTSCALE_HOSTS_BY_REGION =\ + {'eu-west-1': 'osu.eu-west-1.outscale.com', + 'eu-west-2': 'osu.eu-west-2.outscale.com', + 'us-west-1': 'osu.us-west-1.outscale.com', + 'us-east-2': 'osu.us-east-2.outscale.com', + 'cn-southeast-1': 'osu.cn-southeast-1.outscale.hk'} + +S3_RGW_OUTSCALE_DEFAULT_REGION = 'eu-west-2' + + +class S3RGWConnectionAWS4(SignedAWSConnection, BaseS3Connection): + service_name = 's3' + version = API_VERSION + + def __init__(self, user_id, key, secure=True, host=None, port=None, + url=None, timeout=None, proxy_url=None, token=None, + retry_delay=None, backoff=None, **kwargs): + + super(S3RGWConnectionAWS4, self).__init__(user_id, key, + secure, host, + port, url, + timeout, + proxy_url, token, + retry_delay, + backoff, + 4) # force aws4 + + +class S3RGWConnectionAWS2(S3Connection): + + def __init__(self, user_id, key, secure=True, host=None, port=None, + url=None, timeout=None, proxy_url=None, token=None, + retry_delay=None, backoff=None, **kwargs): + + super(S3RGWConnectionAWS2, self).__init__(user_id, key, + secure, host, + port, url, + timeout, + proxy_url, token, + retry_delay, + backoff) + + +class S3RGWStorageDriver(S3StorageDriver): + + def __init__(self, key, secret=None, secure=True, host=None, port=None, + api_version=None, region=S3_RGW_DEFAULT_REGION, **kwargs): + if host is None: + raise LibcloudError('host required', driver=self) + self.name = 'Ceph RGW S3 (%s)' % (region) + self.ex_location_name = region + self.region_name = region + + self.signature_version = str(kwargs.pop('signature_version', + DEFAULT_SIGNATURE_VERSION)) + + if self.signature_version not in ['2', '4']: + raise ValueError('Invalid signature_version: %s' % + (self.signature_version)) + + if self.signature_version == '2': + self.connectionCls = S3RGWOutscaleConnectionAWS2 + elif self.signature_version == '4': + self.connectionCls = S3RGWOutscaleConnectionAWS4 + self.connectionCls.host = host + super(S3RGWStorageDriver, self).__init__(key, secret, + secure, host, port, + api_version, region, + **kwargs) + + def _ex_connection_class_kwargs(self): + kwargs = {} + kwargs['signature_version'] = self.signature_version + return kwargs + + +class S3RGWOutscaleConnectionAWS4(S3RGWConnectionAWS4): + pass + + +class S3RGWOutscaleConnectionAWS2(S3RGWConnectionAWS2): + pass + + +class S3RGWOutscaleStorageDriver(S3RGWStorageDriver): + + def __init__(self, key, secret=None, secure=True, host=None, port=None, + api_version=None, region=S3_RGW_OUTSCALE_DEFAULT_REGION, + **kwargs): + if region not in S3_RGW_OUTSCALE_HOSTS_BY_REGION: + raise LibcloudError('Unknown region (%s)' % (region), driver=self) + + self.name = 'OUTSCALE Ceph RGW S3 (%s)' % (region) + self.ex_location_name = region + self.region_name = region + self.signature_version = str(kwargs.pop('signature_version', + DEFAULT_SIGNATURE_VERSION)) + + if self.signature_version not in ['2', '4']: + raise ValueError('Invalid signature_version: %s' % + (self.signature_version)) + + if self.signature_version == '2': + self.connectionCls = S3RGWOutscaleConnectionAWS2 + elif self.signature_version == '4': + self.connectionCls = S3RGWOutscaleConnectionAWS4 + + host = S3_RGW_OUTSCALE_HOSTS_BY_REGION[region] + self.connectionCls.host = host + super(S3RGWStorageDriver, self).__init__(key, secret, + secure, host, port, + api_version, region, + **kwargs) diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 6a16751a09..26ecd56f90 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -36,7 +36,7 @@ from libcloud.common.types import InvalidCredsError, LibcloudError from libcloud.common.base import ConnectionUserAndKey, RawResponse from libcloud.common.aws import AWSBaseResponse, AWSDriver, \ - AWSTokenConnection, SignedAWSConnection, DEFAULT_SIGNATURE_VERSION + AWSTokenConnection, SignedAWSConnection from libcloud.storage.base import Object, Container, StorageDriver from libcloud.storage.types import ContainerError @@ -60,15 +60,6 @@ S3_AP_NORTHEAST_HOST = S3_AP_NORTHEAST1_HOST S3_SA_EAST_HOST = 's3-sa-east-1.amazonaws.com' -S3_RGW_OUTSCALE_HOSTS_BY_REGION =\ - {'eu-west-1': 'osu.eu-west-1.outscale.com', - 'eu-west-2': 'osu.eu-west-2.outscale.com', - 'us-west-1': 'osu.us-west-1.outscale.com', - 'us-east-2': 'osu.us-east-2.outscale.com', - 'cn-southeast-1': 'osu.cn-southeast-1.outscale.hk'} - -S3_RGW_OUTSCALE_DEFAULT_REGION = 'eu-west-2' - API_VERSION = '2006-03-01' NAMESPACE = 'http://s3.amazonaws.com/doc/%s/' % (API_VERSION) @@ -1013,72 +1004,3 @@ class S3SAEastStorageDriver(S3StorageDriver): name = 'Amazon S3 (sa-east-1)' connectionCls = S3SAEastConnection ex_location_name = 'sa-east-1' - - -class S3RGWOutscaleConnectionAWS4(SignedAWSConnection, BaseS3Connection): - service_name = 's3' - version = API_VERSION - - def __init__(self, user_id, key, secure=True, host=None, port=None, - url=None, timeout=None, proxy_url=None, token=None, - retry_delay=None, backoff=None, **kwargs): - - super(S3RGWOutscaleConnectionAWS4, self).__init__(user_id, key, - secure, host, - port, url, - timeout, - proxy_url, token, - retry_delay, - backoff, - 4) # force aws4 - - -class S3RGWOutscaleConnectionAWS2(S3Connection): - - def __init__(self, user_id, key, secure=True, host=None, port=None, - url=None, timeout=None, proxy_url=None, token=None, - retry_delay=None, backoff=None, **kwargs): - - super(S3RGWOutscaleConnectionAWS2, self).__init__(user_id, key, - secure, host, - port, url, - timeout, - proxy_url, token, - retry_delay, - backoff) - - -class S3RGWOutscaleStorageDriver(S3StorageDriver): - - def __init__(self, key, secret=None, secure=True, host=None, port=None, - api_version=None, region=S3_RGW_OUTSCALE_DEFAULT_REGION, - **kwargs): - if region not in S3_RGW_OUTSCALE_HOSTS_BY_REGION: - raise LibcloudError('Unknown region (%s)' % (region), driver=self) - - self.name = 'OUTSCALE Ceph RGW S3 (%s)' % (region) - self.ex_location_name = region - self.region_name = region - self.signature_version = str(kwargs.pop('signature_version', - DEFAULT_SIGNATURE_VERSION)) - - if self.signature_version not in ['2', '4']: - raise ValueError('Invalid signature_version: %s' % - (self.signature_version)) - - if self.signature_version == '2': - self.connectionCls = S3RGWOutscaleConnectionAWS2 - elif self.signature_version == '4': - self.connectionCls = S3RGWOutscaleConnectionAWS4 - - host = S3_RGW_OUTSCALE_HOSTS_BY_REGION[region] - self.connectionCls.host = host - super(S3RGWOutscaleStorageDriver, self).__init__(key, secret, - secure, host, port, - api_version, region, - **kwargs) - - def _ex_connection_class_kwargs(self): - kwargs = {} - kwargs['signature_version'] = self.signature_version - return kwargs diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py index eddd470cb9..5725a286d9 100644 --- a/libcloud/storage/providers.py +++ b/libcloud/storage/providers.py @@ -43,8 +43,10 @@ ('libcloud.storage.drivers.s3', 'S3APNE2StorageDriver'), Provider.S3_SA_EAST: ('libcloud.storage.drivers.s3', 'S3SAEastStorageDriver'), + Provider.S3_RGW: + ('libcloud.storage.drivers.rgw', 'S3RGWStorageDriver'), Provider.S3_RGW_OUTSCALE: - ('libcloud.storage.drivers.s3', 'S3RGWOutscaleStorageDriver'), + ('libcloud.storage.drivers.rgw', 'S3RGWOutscaleStorageDriver'), Provider.NINEFOLD: ('libcloud.storage.drivers.ninefold', 'NinefoldStorageDriver'), Provider.GOOGLE_STORAGE: diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index c4c61bd897..224bec8a0d 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -49,9 +49,10 @@ class Provider(object): :cvar S3_AP_NORTHEAST_HOST: Amazon S3 Asia South East (Tokyo) :cvar S3_AP_SOUTHEAST_HOST: Amazon S3 Asia South East (Singapore) :cvar S3_EU_WEST: Amazon S3 EU West (Ireland) - :cvar S3_RGW_OUTSCALE: OUTSCALE RGW S3 :cvar S3_US_WEST: Amazon S3 US West (Northern California) :cvar S3_US_WEST_OREGON: Amazon S3 US West 2 (Oregon) + :cvar S3_RGW: S3 RGW + :cvar S3_RGW_OUTSCALE: OUTSCALE S3 RGW """ DUMMY = 'dummy' ALIYUN_OSS = 'aliyun_oss' @@ -71,10 +72,11 @@ class Provider(object): S3_AP_NORTHEAST2 = 's3_ap_northeast_2' S3_AP_SOUTHEAST = 's3_ap_southeast' S3_EU_WEST = 's3_eu_west' - S3_RGW_OUTSCALE = 's3_rgw_outscale' S3_SA_EAST = 's3_sa_east' S3_US_WEST = 's3_us_west' S3_US_WEST_OREGON = 's3_us_west_oregon' + S3_RGW = 's3_rgw' + S3_RGW_OUTSCALE = 's3_rgw_outscale' # Deperecated CLOUDFILES_US = 'cloudfiles_us'