Permalink
Browse files

Merge pull request #3635 from houglum/develop

Add gs support for object-level storage class features.
  • Loading branch information...
2 parents 5e176f5 + f4ad3df commit dc4bf346b55f76f7d8f097fe1461ddfd89a80524 @mfschwartz mfschwartz committed on GitHub Oct 21, 2016
Showing with 119 additions and 82 deletions.
  1. +13 −1 boto/gs/bucket.py
  2. +2 −0 boto/gs/key.py
  3. +67 −70 boto/gs/lifecycle.py
  4. +13 −2 boto/storage_uri.py
  5. +24 −9 tests/integration/gs/test_basic.py
View
@@ -44,11 +44,14 @@
STANDARD_ACL = 'acl'
CORS_ARG = 'cors'
LIFECYCLE_ARG = 'lifecycle'
+STORAGE_CLASS_ARG='storageClass'
ERROR_DETAILS_REGEX = re.compile(r'<Details>(?P<details>.*)</Details>')
class Bucket(S3Bucket):
"""Represents a Google Cloud Storage bucket."""
+ StorageClassBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
+ '<StorageClass>%s</StorageClass>')
VersioningBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<VersioningConfiguration><Status>%s</Status>'
'</VersioningConfiguration>')
@@ -599,7 +602,7 @@ def get_storage_class(self):
:return: The StorageClass for the bucket.
"""
response = self.connection.make_request('GET', self.name,
- query_args='storageClass')
+ query_args=STORAGE_CLASS_ARG)
body = response.read()
if response.status == 200:
rs = ResultSet(self)
@@ -610,6 +613,15 @@ def get_storage_class(self):
raise self.connection.provider.storage_response_error(
response.status, response.reason, body)
+ def set_storage_class(self, storage_class, headers=None):
+ """
+ Sets a bucket's storage class.
+
+ :param str storage_class: A string containing the storage class.
+ :param dict headers: Additional headers to send with the request.
+ """
+ req_body = self.StorageClassBody % (get_utf8_value(storage_class))
+ self.set_subresource(STORAGE_CLASS_ARG, req_body, headers=headers)
# Method with same signature as boto.s3.bucket.Bucket.add_email_grant(),
# to allow polymorphic treatment at application layer.
View
@@ -131,6 +131,8 @@ def handle_addl_headers(self, headers):
self.content_encoding = value
elif key == 'x-goog-stored-content-length':
self.size = int(value)
+ elif key == 'x-goog-storage-class':
+ self.storage_class = value
def open_read(self, headers=None, query_args='',
override_num_retries=None, response_headers=None):
View
@@ -22,44 +22,42 @@
from boto.exception import InvalidLifecycleConfigError
# Relevant tags for the lifecycle configuration XML document.
-LIFECYCLE_CONFIG = 'LifecycleConfiguration'
-RULE = 'Rule'
-ACTION = 'Action'
-DELETE = 'Delete'
-CONDITION = 'Condition'
-AGE = 'Age'
-CREATED_BEFORE = 'CreatedBefore'
-NUM_NEWER_VERSIONS = 'NumberOfNewerVersions'
-IS_LIVE = 'IsLive'
+LIFECYCLE_CONFIG = 'LifecycleConfiguration'
+RULE = 'Rule'
+ACTION = 'Action'
+DELETE = 'Delete'
+SET_STORAGE_CLASS = 'SetStorageClass'
+CONDITION = 'Condition'
+AGE = 'Age'
+CREATED_BEFORE = 'CreatedBefore'
+NUM_NEWER_VERSIONS = 'NumberOfNewerVersions'
+IS_LIVE = 'IsLive'
+MATCHES_STORAGE_CLASS = 'MatchesStorageClass'
# List of all action elements.
-LEGAL_ACTIONS = [DELETE]
-# List of all action parameter elements.
-LEGAL_ACTION_PARAMS = []
+LEGAL_ACTIONS = [DELETE, SET_STORAGE_CLASS]
# List of all condition elements.
-LEGAL_CONDITIONS = [AGE, CREATED_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE]
-# Dictionary mapping actions to supported action parameters for each action.
-LEGAL_ACTION_ACTION_PARAMS = {
- DELETE: [],
-}
+LEGAL_CONDITIONS = [AGE, CREATED_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE,
+ MATCHES_STORAGE_CLASS]
+# List of conditions elements that may be repeated.
+LEGAL_REPEATABLE_CONDITIONS = [MATCHES_STORAGE_CLASS]
class Rule(object):
"""
A lifecycle rule for a bucket.
:ivar action: Action to be taken.
- :ivar action_params: A dictionary of action specific parameters. Each item
- in the dictionary represents the name and value of an action parameter.
+ :ivar action_text: The text value for the specified action, if any.
:ivar conditions: A dictionary of conditions that specify when the action
- should be taken. Each item in the dictionary represents the name and value
- of a condition.
+ should be taken. Each item in the dictionary represents the name and
+ value (or a list of multiple values, if applicable) of a condition.
"""
- def __init__(self, action=None, action_params=None, conditions=None):
+ def __init__(self, action=None, action_text=None, conditions=None):
self.action = action
- self.action_params = action_params or {}
+ self.action_text = action_text
self.conditions = conditions or {}
# Name of the current enclosing tag (used to validate the schema).
@@ -88,23 +86,15 @@ def startElement(self, name, attrs, connection):
raise InvalidLifecycleConfigError(
'Only one action tag is allowed in each rule')
self.action = name
- elif name in LEGAL_ACTION_PARAMS:
- # Make sure this tag is found in an action tag.
- if self.current_tag not in LEGAL_ACTIONS:
- raise InvalidLifecycleConfigError(
- 'Tag %s found outside of action' % name)
- # Make sure this tag is allowed for the current action tag.
- if name not in LEGAL_ACTION_ACTION_PARAMS[self.action]:
- raise InvalidLifecycleConfigError(
- 'Tag %s not allowed in action %s' % (name, self.action))
elif name == CONDITION:
self.validateStartTag(name, RULE)
elif name in LEGAL_CONDITIONS:
self.validateStartTag(name, CONDITION)
# Verify there is no duplicate conditions.
- if name in self.conditions:
+ if (name in self.conditions and
+ name not in LEGAL_REPEATABLE_CONDITIONS):
raise InvalidLifecycleConfigError(
- 'Found duplicate conditions %s' % name)
+ 'Found duplicate non-repeatable conditions %s' % name)
else:
raise InvalidLifecycleConfigError('Unsupported tag ' + name)
self.current_tag = name
@@ -118,17 +108,20 @@ def endElement(self, name, value, connection):
elif name == ACTION:
self.current_tag = RULE
elif name in LEGAL_ACTIONS:
+ if name == SET_STORAGE_CLASS and value is not None:
+ self.action_text = value.strip()
self.current_tag = ACTION
- elif name in LEGAL_ACTION_PARAMS:
- self.current_tag = self.action
- # Add the action parameter name and value to the dictionary.
- self.action_params[name] = value.strip()
elif name == CONDITION:
self.current_tag = RULE
elif name in LEGAL_CONDITIONS:
self.current_tag = CONDITION
- # Add the condition name and value to the dictionary.
- self.conditions[name] = value.strip()
+ # Some conditions specify a list of values.
+ if name in LEGAL_REPEATABLE_CONDITIONS:
+ if name not in self.conditions:
+ self.conditions[name] = []
+ self.conditions[name].append(value.strip())
+ else:
+ self.conditions[name] = value.strip()
else:
raise InvalidLifecycleConfigError('Unsupported end tag ' + name)
@@ -143,26 +136,32 @@ def validate(self):
def to_xml(self):
"""Convert the rule into XML string representation."""
- s = '<' + RULE + '>'
- s += '<' + ACTION + '>'
- if self.action_params:
- s += '<' + self.action + '>'
- for param in LEGAL_ACTION_PARAMS:
- if param in self.action_params:
- s += ('<' + param + '>' + self.action_params[param] + '</'
- + param + '>')
- s += '</' + self.action + '>'
+ s = ['<' + RULE + '>']
+ s.append('<' + ACTION + '>')
+ if self.action_text:
+ s.extend(['<' + self.action + '>',
+ self.action_text,
+ '</' + self.action + '>'])
else:
- s += '<' + self.action + '/>'
- s += '</' + ACTION + '>'
- s += '<' + CONDITION + '>'
- for condition in LEGAL_CONDITIONS:
- if condition in self.conditions:
- s += ('<' + condition + '>' + self.conditions[condition] + '</'
- + condition + '>')
- s += '</' + CONDITION + '>'
- s += '</' + RULE + '>'
- return s
+ s.append('<' + self.action + '/>')
+ s.append('</' + ACTION + '>')
+ s.append('<' + CONDITION + '>')
+ for condition_name in self.conditions:
+ if condition_name not in LEGAL_CONDITIONS:
+ continue
+ if condition_name in LEGAL_REPEATABLE_CONDITIONS:
+ condition_values = self.conditions[condition_name]
+ else:
+ # Wrap condition value in a list, allowing us to iterate over
+ # all condition values using the same logic.
+ condition_values = [self.conditions[condition_name]]
+ for condition_value in condition_values:
+ s.extend(['<' + condition_name + '>',
+ condition_value,
+ '</' + condition_name + '>'])
+ s.append('</' + CONDITION + '>')
+ s.append('</' + RULE + '>')
+ return ''.join(s)
class LifecycleConfig(list):
"""
@@ -196,14 +195,14 @@ def endElement(self, name, value, connection):
def to_xml(self):
"""Convert LifecycleConfig object into XML string representation."""
- s = '<?xml version="1.0" encoding="UTF-8"?>'
- s += '<' + LIFECYCLE_CONFIG + '>'
+ s = ['<?xml version="1.0" encoding="UTF-8"?>']
+ s.append('<' + LIFECYCLE_CONFIG + '>')
for rule in self:
- s += rule.to_xml()
- s += '</' + LIFECYCLE_CONFIG + '>'
- return s
+ s.append(rule.to_xml())
+ s.append('</' + LIFECYCLE_CONFIG + '>')
+ return ''.join(s)
- def add_rule(self, action, action_params, conditions):
+ def add_rule(self, action, action_text, conditions):
"""
Add a rule to this Lifecycle configuration. This only adds the rule to
the local copy. To install the new rule(s) on the bucket, you need to
@@ -213,15 +212,13 @@ def add_rule(self, action, action_params, conditions):
:type action: str
:param action: Action to be taken.
- :type action_params: dict
- :param action_params: A dictionary of action specific parameters. Each
- item in the dictionary represents the name and value of an action
- parameter.
+ :type action_text: str
+ :param action_text: Value for the specified action.
:type conditions: dict
:param conditions: A dictionary of conditions that specify when the
action should be taken. Each item in the dictionary represents the name
and value of a condition.
"""
- rule = Rule(action, action_params, conditions)
+ rule = Rule(action, action_text, conditions)
self.append(rule)
View
@@ -444,14 +444,25 @@ def get_location(self, validate=False, headers=None):
def get_storage_class(self, validate=False, headers=None):
self._check_bucket_uri('get_storage_class')
- # StorageClass is defined as a bucket param for GCS, but as a key
- # param for S3.
+ # StorageClass is defined as a bucket and object param for GCS, but
+ # only as a key param for S3.
if self.scheme != 'gs':
raise ValueError('get_storage_class() not supported for %s '
'URIs.' % self.scheme)
bucket = self.get_bucket(validate, headers)
return bucket.get_storage_class()
+ def set_storage_class(self, storage_class, validate=False, headers=None):
+ """Updates a bucket's storage class."""
+ self._check_bucket_uri('set_storage_class')
+ # StorageClass is defined as a bucket and object param for GCS, but
+ # only as a key param for S3.
+ if self.scheme != 'gs':
+ raise ValueError('set_storage_class() not supported for %s '
+ 'URIs.' % self.scheme)
+ bucket = self.get_bucket(validate, headers)
+ bucket.set_storage_class(storage_class, headers)
+
def get_subresource(self, subresource, validate=False, headers=None,
version_id=None):
self._check_bucket_uri('get_subresource')
@@ -56,15 +56,22 @@
LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>'
'<LifecycleConfiguration><Rule>'
'<Action><Delete/></Action>'
- '<Condition><Age>365</Age>'
+ '<Condition>''<IsLive>true</IsLive>'
+ '<MatchesStorageClass>STANDARD</MatchesStorageClass>'
+ '<Age>365</Age>'
'<CreatedBefore>2013-01-15</CreatedBefore>'
'<NumberOfNewerVersions>3</NumberOfNewerVersions>'
- '<IsLive>true</IsLive></Condition>'
- '</Rule></LifecycleConfiguration>')
-LIFECYCLE_CONDITIONS = {'Age': '365',
- 'CreatedBefore': '2013-01-15',
- 'NumberOfNewerVersions': '3',
- 'IsLive': 'true'}
+ '</Condition></Rule><Rule>'
+ '<Action><SetStorageClass>NEARLINE</SetStorageClass></Action>'
+ '<Condition><Age>366</Age>'
+ '</Condition></Rule></LifecycleConfiguration>')
+LIFECYCLE_CONDITIONS_FOR_DELETE_RULE = {
+ 'Age': '365',
+ 'CreatedBefore': '2013-01-15',
+ 'NumberOfNewerVersions': '3',
+ 'IsLive': 'true',
+ 'MatchesStorageClass': ['STANDARD']}
+LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE = {'Age': '366'}
# Regexp for matching project-private default object ACL.
PROJECT_PRIVATE_RE = ('\s*<AccessControlList>\s*<Entries>\s*<Entry>'
@@ -412,7 +419,11 @@ def test_lifecycle_config_bucket(self):
self.assertEqual(xml, LIFECYCLE_EMPTY)
# set lifecycle config
lifecycle_config = LifecycleConfig()
- lifecycle_config.add_rule('Delete', None, LIFECYCLE_CONDITIONS)
+ lifecycle_config.add_rule(
+ 'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE)
+ lifecycle_config.add_rule(
+ 'SetStorageClass', 'NEARLINE',
+ LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE)
bucket.configure_lifecycle(lifecycle_config)
xml = bucket.get_lifecycle_config().to_xml()
self.assertEqual(xml, LIFECYCLE_DOC)
@@ -428,7 +439,11 @@ def test_lifecycle_config_storage_uri(self):
self.assertEqual(xml, LIFECYCLE_EMPTY)
# set lifecycle config
lifecycle_config = LifecycleConfig()
- lifecycle_config.add_rule('Delete', None, LIFECYCLE_CONDITIONS)
+ lifecycle_config.add_rule(
+ 'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE)
+ lifecycle_config.add_rule(
+ 'SetStorageClass', 'NEARLINE',
+ LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE)
uri.configure_lifecycle(lifecycle_config)
xml = uri.get_lifecycle_config().to_xml()
self.assertEqual(xml, LIFECYCLE_DOC)

0 comments on commit dc4bf34

Please sign in to comment.