Merge pull request #3635 from houglum/develop

Add gs support for object-level storage class features.

boto/gs/bucket.py

@@ -44,11 +44,14 @@
 STANDARD_ACL = 'acl'
 CORS_ARG = 'cors'
 LIFECYCLE_ARG = 'lifecycle'
+STORAGE_CLASS_ARG='storageClass'
 ERROR_DETAILS_REGEX = re.compile(r'<Details>(?P<details>.*)</Details>')
 
 class Bucket(S3Bucket):
     """Represents a Google Cloud Storage bucket."""
 
+    StorageClassBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
+                        '<StorageClass>%s</StorageClass>')
     VersioningBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
                       '<VersioningConfiguration><Status>%s</Status>'
                       '</VersioningConfiguration>')
@@ -599,7 +602,7 @@ def get_storage_class(self):
         :return: The StorageClass for the bucket.
         """
         response = self.connection.make_request('GET', self.name,
-                                                query_args='storageClass')
+                                                query_args=STORAGE_CLASS_ARG)
         body = response.read()
         if response.status == 200:
             rs = ResultSet(self)
@@ -610,6 +613,15 @@ def get_storage_class(self):
             raise self.connection.provider.storage_response_error(
                 response.status, response.reason, body)
 
+    def set_storage_class(self, storage_class, headers=None):
+        """
+        Sets a bucket's storage class.
+
+        :param str storage_class: A string containing the storage class.
+        :param dict headers: Additional headers to send with the request.
+        """
+        req_body = self.StorageClassBody % (get_utf8_value(storage_class))
+        self.set_subresource(STORAGE_CLASS_ARG, req_body, headers=headers)
 
     # Method with same signature as boto.s3.bucket.Bucket.add_email_grant(),
     # to allow polymorphic treatment at application layer.

boto/gs/key.py

@@ -131,6 +131,8 @@ def handle_addl_headers(self, headers):
                 self.content_encoding = value
             elif key == 'x-goog-stored-content-length':
                 self.size = int(value)
+            elif key == 'x-goog-storage-class':
+                self.storage_class = value
 
     def open_read(self, headers=None, query_args='',
                   override_num_retries=None, response_headers=None):

boto/gs/lifecycle.py

@@ -22,44 +22,42 @@
 from boto.exception import InvalidLifecycleConfigError
 
 # Relevant tags for the lifecycle configuration XML document.
-LIFECYCLE_CONFIG   = 'LifecycleConfiguration'
-RULE               = 'Rule'
-ACTION             = 'Action'
-DELETE             = 'Delete'
-CONDITION          = 'Condition'
-AGE                = 'Age'
-CREATED_BEFORE     = 'CreatedBefore'
-NUM_NEWER_VERSIONS = 'NumberOfNewerVersions'
-IS_LIVE            = 'IsLive'
+LIFECYCLE_CONFIG      = 'LifecycleConfiguration'
+RULE                  = 'Rule'
+ACTION                = 'Action'
+DELETE                = 'Delete'
+SET_STORAGE_CLASS     = 'SetStorageClass'
+CONDITION             = 'Condition'
+AGE                   = 'Age'
+CREATED_BEFORE        = 'CreatedBefore'
+NUM_NEWER_VERSIONS    = 'NumberOfNewerVersions'
+IS_LIVE               = 'IsLive'
+MATCHES_STORAGE_CLASS = 'MatchesStorageClass'
 
 # List of all action elements.
-LEGAL_ACTIONS = [DELETE]
-# List of all action parameter elements.
-LEGAL_ACTION_PARAMS = []
+LEGAL_ACTIONS = [DELETE, SET_STORAGE_CLASS]
 # List of all condition elements.
-LEGAL_CONDITIONS = [AGE, CREATED_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE]
-# Dictionary mapping actions to supported action parameters for each action.
-LEGAL_ACTION_ACTION_PARAMS = {
-    DELETE: [],
-}
+LEGAL_CONDITIONS = [AGE, CREATED_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE,
+                    MATCHES_STORAGE_CLASS]
+# List of conditions elements that may be repeated.
+LEGAL_REPEATABLE_CONDITIONS = [MATCHES_STORAGE_CLASS]
 
 class Rule(object):
     """
     A lifecycle rule for a bucket.
 
     :ivar action: Action to be taken.
 
-    :ivar action_params: A dictionary of action specific parameters. Each item
-    in the dictionary represents the name and value of an action parameter.
+    :ivar action_text: The text value for the specified action, if any.
 
     :ivar conditions: A dictionary of conditions that specify when the action
-    should be taken. Each item in the dictionary represents the name and value
-    of a condition.
+    should be taken. Each item in the dictionary represents the name and
+    value (or a list of multiple values, if applicable) of a condition.
     """
 
-    def __init__(self, action=None, action_params=None, conditions=None):
+    def __init__(self, action=None, action_text=None, conditions=None):
         self.action = action
-        self.action_params = action_params or {}
+        self.action_text = action_text
         self.conditions = conditions or {}
 
         # Name of the current enclosing tag (used to validate the schema).
@@ -88,23 +86,15 @@ def startElement(self, name, attrs, connection):
                 raise InvalidLifecycleConfigError(
                     'Only one action tag is allowed in each rule')
             self.action = name
-        elif name in LEGAL_ACTION_PARAMS:
-            # Make sure this tag is found in an action tag.
-            if self.current_tag not in LEGAL_ACTIONS:
-                raise InvalidLifecycleConfigError(
-                    'Tag %s found outside of action' % name)
-            # Make sure this tag is allowed for the current action tag.
-            if name not in LEGAL_ACTION_ACTION_PARAMS[self.action]:
-                raise InvalidLifecycleConfigError(
-                    'Tag %s not allowed in action %s' % (name, self.action))
         elif name == CONDITION:
             self.validateStartTag(name, RULE)
         elif name in LEGAL_CONDITIONS:
             self.validateStartTag(name, CONDITION)
             # Verify there is no duplicate conditions.
-            if name in self.conditions:
+            if (name in self.conditions and
+                name not in LEGAL_REPEATABLE_CONDITIONS):
                 raise InvalidLifecycleConfigError(
-                    'Found duplicate conditions %s' % name)
+                    'Found duplicate non-repeatable conditions %s' % name)
         else:
             raise InvalidLifecycleConfigError('Unsupported tag ' + name)
         self.current_tag = name
@@ -118,17 +108,20 @@ def endElement(self, name, value, connection):
         elif name == ACTION:
             self.current_tag = RULE
         elif name in LEGAL_ACTIONS:
+            if name == SET_STORAGE_CLASS and value is not None:
+                self.action_text = value.strip()
             self.current_tag = ACTION
-        elif name in LEGAL_ACTION_PARAMS:
-            self.current_tag = self.action
-            # Add the action parameter name and value to the dictionary.
-            self.action_params[name] = value.strip()
         elif name == CONDITION:
             self.current_tag = RULE
         elif name in LEGAL_CONDITIONS:
             self.current_tag = CONDITION
-            # Add the condition name and value to the dictionary.
-            self.conditions[name] = value.strip()
+            # Some conditions specify a list of values.
+            if name in LEGAL_REPEATABLE_CONDITIONS:
+                if name not in self.conditions:
+                    self.conditions[name] = []
+                self.conditions[name].append(value.strip())
+            else:
+                self.conditions[name] = value.strip()
         else:
             raise InvalidLifecycleConfigError('Unsupported end tag ' + name)
 
@@ -143,26 +136,32 @@ def validate(self):
 
     def to_xml(self):
         """Convert the rule into XML string representation."""
-        s = '<' + RULE + '>'
-        s += '<' + ACTION + '>'
-        if self.action_params:
-            s += '<' + self.action + '>'
-            for param in LEGAL_ACTION_PARAMS:
-                if param in self.action_params:
-                    s += ('<' + param + '>' + self.action_params[param] + '</'
-                          + param + '>')
-            s += '</' + self.action + '>'
+        s = ['<' + RULE + '>']
+        s.append('<' + ACTION + '>')
+        if self.action_text:
+            s.extend(['<' + self.action + '>',
+                      self.action_text,
+                      '</' + self.action + '>'])
         else:
-            s += '<' + self.action + '/>'
-        s += '</' + ACTION + '>'
-        s += '<' + CONDITION + '>'
-        for condition in LEGAL_CONDITIONS:
-            if condition in self.conditions:
-                s += ('<' + condition + '>' + self.conditions[condition] + '</'
-                      + condition + '>')
-        s += '</' + CONDITION + '>'
-        s += '</' + RULE + '>'
-        return s
+            s.append('<' + self.action + '/>')
+        s.append('</' + ACTION + '>')
+        s.append('<' + CONDITION + '>')
+        for condition_name in self.conditions:
+            if condition_name not in LEGAL_CONDITIONS:
+                continue
+            if condition_name in LEGAL_REPEATABLE_CONDITIONS:
+                condition_values = self.conditions[condition_name]
+            else:
+                # Wrap condition value in a list, allowing us to iterate over
+                # all condition values using the same logic.
+                condition_values = [self.conditions[condition_name]]
+            for condition_value in condition_values:
+                s.extend(['<' + condition_name + '>',
+                          condition_value,
+                          '</' + condition_name + '>'])
+        s.append('</' + CONDITION + '>')
+        s.append('</' + RULE + '>')
+        return ''.join(s)
 
 class LifecycleConfig(list):
     """
@@ -196,14 +195,14 @@ def endElement(self, name, value, connection):
 
     def to_xml(self):
         """Convert LifecycleConfig object into XML string representation."""
-        s = '<?xml version="1.0" encoding="UTF-8"?>'
-        s += '<' + LIFECYCLE_CONFIG + '>'
+        s = ['<?xml version="1.0" encoding="UTF-8"?>']
+        s.append('<' + LIFECYCLE_CONFIG + '>')
         for rule in self:
-            s += rule.to_xml()
-        s += '</' + LIFECYCLE_CONFIG + '>'
-        return s
+            s.append(rule.to_xml())
+        s.append('</' + LIFECYCLE_CONFIG + '>')
+        return ''.join(s)
 
-    def add_rule(self, action, action_params, conditions):
+    def add_rule(self, action, action_text, conditions):
         """
         Add a rule to this Lifecycle configuration.  This only adds the rule to
         the local copy.  To install the new rule(s) on the bucket, you need to
@@ -213,15 +212,13 @@ def add_rule(self, action, action_params, conditions):
         :type action: str
         :param action: Action to be taken.
 
-        :type action_params: dict
-        :param action_params: A dictionary of action specific parameters. Each
-        item in the dictionary represents the name and value of an action
-        parameter.
+        :type action_text: str
+        :param action_text: Value for the specified action.
 
         :type conditions: dict
         :param conditions: A dictionary of conditions that specify when the
         action should be taken. Each item in the dictionary represents the name
         and value of a condition.
         """
-        rule = Rule(action, action_params, conditions)
+        rule = Rule(action, action_text, conditions)
         self.append(rule)

boto/storage_uri.py

@@ -444,14 +444,25 @@ def get_location(self, validate=False, headers=None):
 
     def get_storage_class(self, validate=False, headers=None):
         self._check_bucket_uri('get_storage_class')
-        # StorageClass is defined as a bucket param for GCS, but as a key
-        # param for S3.
+        # StorageClass is defined as a bucket and object param for GCS, but
+        # only as a key param for S3.
         if self.scheme != 'gs':
             raise ValueError('get_storage_class() not supported for %s '
                              'URIs.' % self.scheme)
         bucket = self.get_bucket(validate, headers)
         return bucket.get_storage_class()
 
+    def set_storage_class(self, storage_class, validate=False, headers=None):
+        """Updates a bucket's storage class."""
+        self._check_bucket_uri('set_storage_class')
+        # StorageClass is defined as a bucket and object param for GCS, but
+        # only as a key param for S3.
+        if self.scheme != 'gs':
+            raise ValueError('set_storage_class() not supported for %s '
+                             'URIs.' % self.scheme)
+        bucket = self.get_bucket(validate, headers)
+        bucket.set_storage_class(storage_class, headers)
+
     def get_subresource(self, subresource, validate=False, headers=None,
                         version_id=None):
         self._check_bucket_uri('get_subresource')

tests/integration/gs/test_basic.py

@@ -56,15 +56,22 @@
 LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>'
                  '<LifecycleConfiguration><Rule>'
                  '<Action><Delete/></Action>'
-                 '<Condition><Age>365</Age>'
+                 '<Condition>''<IsLive>true</IsLive>'
+                 '<MatchesStorageClass>STANDARD</MatchesStorageClass>'
+                 '<Age>365</Age>'
                  '<CreatedBefore>2013-01-15</CreatedBefore>'
                  '<NumberOfNewerVersions>3</NumberOfNewerVersions>'
-                 '<IsLive>true</IsLive></Condition>'
-                 '</Rule></LifecycleConfiguration>')
-LIFECYCLE_CONDITIONS = {'Age': '365',
-                        'CreatedBefore': '2013-01-15',
-                        'NumberOfNewerVersions': '3',
-                        'IsLive': 'true'}
+                 '</Condition></Rule><Rule>'
+                 '<Action><SetStorageClass>NEARLINE</SetStorageClass></Action>'
+                 '<Condition><Age>366</Age>'
+                 '</Condition></Rule></LifecycleConfiguration>')
+LIFECYCLE_CONDITIONS_FOR_DELETE_RULE = {
+    'Age': '365',
+    'CreatedBefore': '2013-01-15',
+    'NumberOfNewerVersions': '3',
+    'IsLive': 'true',
+    'MatchesStorageClass': ['STANDARD']}
+LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE = {'Age': '366'}
 
 # Regexp for matching project-private default object ACL.
 PROJECT_PRIVATE_RE = ('\s*<AccessControlList>\s*<Entries>\s*<Entry>'
@@ -412,7 +419,11 @@ def test_lifecycle_config_bucket(self):
         self.assertEqual(xml, LIFECYCLE_EMPTY)
         # set lifecycle config
         lifecycle_config = LifecycleConfig()
-        lifecycle_config.add_rule('Delete', None, LIFECYCLE_CONDITIONS)
+        lifecycle_config.add_rule(
+            'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE)
+        lifecycle_config.add_rule(
+            'SetStorageClass', 'NEARLINE',
+            LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE)
         bucket.configure_lifecycle(lifecycle_config)
         xml = bucket.get_lifecycle_config().to_xml()
         self.assertEqual(xml, LIFECYCLE_DOC)
@@ -428,7 +439,11 @@ def test_lifecycle_config_storage_uri(self):
         self.assertEqual(xml, LIFECYCLE_EMPTY)
         # set lifecycle config
         lifecycle_config = LifecycleConfig()
-        lifecycle_config.add_rule('Delete', None, LIFECYCLE_CONDITIONS)
+        lifecycle_config.add_rule(
+            'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE)
+        lifecycle_config.add_rule(
+            'SetStorageClass', 'NEARLINE',
+            LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE)
         uri.configure_lifecycle(lifecycle_config)
         xml = uri.get_lifecycle_config().to_xml()
         self.assertEqual(xml, LIFECYCLE_DOC)