Merge branch 'ryansydnor-s3' into develop

* ryansydnor-s3:
  Allow s3 bucket lifecycle policies with multiple transitions

Author: jamesls

boto/s3/lifecycle.py

@@ -54,14 +54,21 @@ def __init__(self, id=None, prefix=None, status=None, expiration=None,
         else:
             # None or object
             self.expiration = expiration
-        self.transition = transition
+
+        # retain backwards compatibility
+        if isinstance(transition, Transition):
+            self.transition = Transitions()
+            self.transition.append(transition)
+        elif transition:
+            self.transition = transition
+        else:
+            self.transition = Transitions()
 
     def __repr__(self):
         return '<Rule: %s>' % self.id
 
     def startElement(self, name, attrs, connection):
         if name == 'Transition':
-            self.transition = Transition()
             return self.transition
         elif name == 'Expiration':
             self.expiration = Expiration()
@@ -139,25 +146,13 @@ class Transition(object):
         in ISO 8601 format.
 
     :ivar storage_class: The storage class to transition to.  Valid
-        values are GLACIER.
-
+        values are GLACIER, STANDARD_IA.
     """
     def __init__(self, days=None, date=None, storage_class=None):
         self.days = days
         self.date = date
         self.storage_class = storage_class
 
-    def startElement(self, name, attrs, connection):
-        return None
-
-    def endElement(self, name, value, connection):
-        if name == 'Days':
-            self.days = int(value)
-        elif name == 'Date':
-            self.date = value
-        elif name == 'StorageClass':
-            self.storage_class = value
-
     def __repr__(self):
         if self.days is None:
             how_long = "on: %s" % self.date
@@ -175,6 +170,86 @@ def to_xml(self):
         s += '</Transition>'
         return s
 
+class Transitions(list):
+    """
+    A container for the transitions associated with a Lifecycle's Rule configuration.
+    """
+    def __init__(self):
+        self.transition_properties = 3
+        self.current_transition_property = 1
+        self.temp_days = None
+        self.temp_date = None
+        self.temp_storage_class = None
+
+    def startElement(self, name, attrs, connection):
+        return None
+
+    def endElement(self, name, value, connection):
+        if name == 'Days':
+            self.temp_days = int(value)
+        elif name == 'Date':
+            self.temp_date = value
+        elif name == 'StorageClass':
+            self.temp_storage_class = value
+
+        # the XML does not contain a <Transitions> tag
+        # but rather N number of <Transition> tags not
+        # structured in any sort of hierarchy.
+        if self.current_transition_property == self.transition_properties:
+            self.append(Transition(self.temp_days, self.temp_date, self.temp_storage_class))
+            self.temp_days = self.temp_date = self.temp_storage_class = None
+            self.current_transition_property = 1
+        else:
+            self.current_transition_property += 1
+
+    def to_xml(self):
+        """
+        Returns a string containing the XML version of the Lifecycle
+        configuration as defined by S3.
+        """
+        s = ''
+        for transition in self:
+            s += transition.to_xml()
+        return s
+
+    def add_transition(self, days=None, date=None, storage_class=None):
+        """
+        Add a transition to this Lifecycle configuration.  This only adds
+        the rule to the local copy.  To install the new rule(s) on
+        the bucket, you need to pass this Lifecycle config object
+        to the configure_lifecycle method of the Bucket object.
+
+        :ivar days: The number of days until the object should be moved.
+
+        :ivar date: The date when the object should be moved.  Should be
+            in ISO 8601 format.
+
+        :ivar storage_class: The storage class to transition to.  Valid
+            values are GLACIER, STANDARD_IA.
+        """
+        transition = Transition(days, date, storage_class)
+        self.append(transition)
+
+    def __first_or_default(self, prop):
+        for transition in self:
+            return getattr(transition, prop)
+        return None
+
+    # maintain backwards compatibility so that we can continue utilizing
+    # 'rule.transition.days' syntax
+    @property
+    def days(self):
+        return self.__first_or_default('days')
+
+    @property
+    def date(self):
+        return self.__first_or_default('date')
+
+    @property
+    def storage_class(self):
+        return self.__first_or_default('storage_class')
+
+
 class Lifecycle(list):
     """
     A container for the rules associated with a Lifecycle configuration.
@@ -228,7 +303,7 @@ def add_rule(self, id=None, prefix='', status='Enabled',
             that are subject to the rule. The value must be a non-zero
             positive integer. A Expiration object instance is also perfect.
 
-        :type transition: Transition
+        :type transition: Transitions
         :param transition: Indicates when an object transitions to a
             different storage class. 
         """

docs/source/s3_tut.rst

@@ -442,33 +442,38 @@ And, finally, to delete all CORS configurations from a bucket::
 
     >>> bucket.delete_cors()
 
-Transitioning Objects to Glacier
+Transitioning Objects
 --------------------------------
 
-You can configure objects in S3 to transition to Glacier after a period of
-time.  This is done using lifecycle policies.  A lifecycle policy can also
-specify that an object should be deleted after a period of time.  Lifecycle
-configurations are assigned to buckets and require these parameters:
+S3 buckets support transitioning objects to various storage classes. This is
+done using lifecycle policies. You can currently transitions objects to 
+Infrequent Access, Glacier, or just plain Expire. All of these options are 
+capable of being applied after a number of days or after a given date.
+Lifecycle configurations are assigned to buckets and require these parameters:
 
-* The object prefix that identifies the objects you are targeting.
+* The object prefix that identifies the objects you are targeting. (or none)
 * The action you want S3 to perform on the identified objects.
-* The date (or time period) when you want S3 to perform these actions.
+* The date or number of days when you want S3 to perform these actions.
 
-For example, given a bucket ``s3-glacier-boto-demo``, we can first retrieve the
+For example, given a bucket ``s3-lifecycle-boto-demo``, we can first retrieve the
 bucket::
 
     >>> import boto
     >>> c = boto.connect_s3()
-    >>> bucket = c.get_bucket('s3-glacier-boto-demo')
+    >>> bucket = c.get_bucket('s3-lifecycle-boto-demo')
 
 Then we can create a lifecycle object.  In our example, we want all objects
-under ``logs/*`` to transition to Glacier 30 days after the object is created.
+under ``logs/*`` to transition to Standard IA 30 days after the object is created,
+glacier 90 days after creation, and be deleted 120 days after creation.
 
 ::
 
-    >>> from boto.s3.lifecycle import Lifecycle, Transition, Rule
-    >>> to_glacier = Transition(days=30, storage_class='GLACIER')
-    >>> rule = Rule('ruleid', 'logs/', 'Enabled', transition=to_glacier)
+    >>> from boto.s3.lifecycle import Lifecycle, Transitions, Rule
+    >>> transitions = Transitions()
+    >>> transitions.add_transition(days=30, storage_class='STANDARD_IA')
+    >>> transitions.add_transition(days=90, storage_class='GLACIER')
+    >>> expiration = Expiration(days=120)
+    >>> rule = Rule(id='ruleid', prefix='logs/', status='Enabled', expiration=expiration, transition=transitions)
     >>> lifecycle = Lifecycle()
     >>> lifecycle.append(rule)
 
@@ -485,19 +490,27 @@ You can also retrieve the current lifecycle policy for the bucket::
 
     >>> current = bucket.get_lifecycle_config()
     >>> print current[0].transition
-    <Transition: in: 30 days, GLACIER>
+    >>> print current[0].expiration
+    [<Transition: in: 90 days, GLACIER>, <Transition: in: 30 days, STANDARD_IA>]
+    <Expiration: in: 120 days>
+
+Note: We have deprecated directly accessing transition properties from the lifecycle
+object. You must index into the transition array first.
 
-When an object transitions to Glacier, the storage class will be
+When an object transitions, the storage class will be
 updated.  This can be seen when you **list** the objects in a bucket::
 
     >>> for key in bucket.list():
     ...   print key, key.storage_class
     ...
-    <Key: s3-glacier-boto-demo,logs/testlog1.log> GLACIER
+    <Key: s3-lifecycle-boto-demo,logs/testlog1.log> STANDARD_IA
+    <Key: s3-lifecycle-boto-demo,logs/testlog2.log> GLACIER
 
 You can also use the prefix argument to the ``bucket.list`` method::
 
     >>> print list(b.list(prefix='logs/testlog1.log'))[0].storage_class
+    >>> print list(b.list(prefix='logs/testlog2.log'))[0].storage_class
+    u'STANDARD_IA'
     u'GLACIER'
 
 

tests/unit/s3/test_lifecycle.py

@@ -50,27 +50,103 @@ def default_body(self):
             <Status>Disabled</Status>
             <Transition>
               <Date>2012-12-31T00:00:000Z</Date>
-              <StorageClass>GLACIER</StorageClass>
+              <StorageClass>STANDARD_IA</StorageClass>
+            </Transition>
+            <Expiration>
+              <Date>2012-12-31T00:00:000Z</Date>
+            </Expiration>
+          </Rule>
+          <Rule>
+            <ID>multiple-transitions</ID>
+            <Prefix></Prefix>
+            <Status>Enabled</Status>
+            <Transition>
+                <Days>30</Days>
+                <StorageClass>STANDARD_IA</StorageClass>
+            </Transition>
+            <Transition>
+                <Days>90</Days>
+                <StorageClass>GLACIER</StorageClass>
             </Transition>
           </Rule>
         </LifecycleConfiguration>
         """
 
-    def test_parse_lifecycle_response(self):
+    def _get_bucket_lifecycle_config(self):
         self.set_http_response(status_code=200)
         bucket = Bucket(self.service_connection, 'mybucket')
-        response = bucket.get_lifecycle_config()
-        self.assertEqual(len(response), 2)
-        rule = response[0]
+        return bucket.get_lifecycle_config()
+
+    def test_lifecycle_response_contains_all_rules(self):
+        self.assertEqual(len(self._get_bucket_lifecycle_config()), 3)
+
+    def test_parse_lifecycle_id(self):
+        rule = self._get_bucket_lifecycle_config()[0]
         self.assertEqual(rule.id, 'rule-1')
+
+    def test_parse_lifecycle_prefix(self):
+        rule = self._get_bucket_lifecycle_config()[0]
         self.assertEqual(rule.prefix, 'prefix/foo')
+
+    def test_parse_lifecycle_no_prefix(self):
+        rule = self._get_bucket_lifecycle_config()[2]
+        self.assertEquals(rule.prefix, '')
+
+    def test_parse_lifecycle_enabled(self):
+        rule = self._get_bucket_lifecycle_config()[0]
         self.assertEqual(rule.status, 'Enabled')
+
+    def test_parse_lifecycle_disabled(self):
+        rule = self._get_bucket_lifecycle_config()[1]
+        self.assertEqual(rule.status, 'Disabled')
+
+    def test_parse_expiration_days(self):
+        rule = self._get_bucket_lifecycle_config()[0]
         self.assertEqual(rule.expiration.days, 365)
-        self.assertIsNone(rule.expiration.date)
-        transition = rule.transition
-        self.assertEqual(transition.days, 30)
+
+    def test_parse_expiration_date(self):
+        rule = self._get_bucket_lifecycle_config()[1]
+        self.assertEqual(rule.expiration.date, '2012-12-31T00:00:000Z')
+
+    def test_parse_expiration_not_required(self):
+        rule = self._get_bucket_lifecycle_config()[2]
+        self.assertIsNone(rule.expiration)
+
+    def test_parse_transition_days(self):
+        transition = self._get_bucket_lifecycle_config()[0].transition[0]
+        self.assertEquals(transition.days, 30)
+        self.assertIsNone(transition.date)
+
+    def test_parse_transition_days_deprecated(self):
+        transition = self._get_bucket_lifecycle_config()[0].transition
+        self.assertEquals(transition.days, 30)
+        self.assertIsNone(transition.date)
+
+    def test_parse_transition_date(self):
+        transition = self._get_bucket_lifecycle_config()[1].transition[0]
+        self.assertEquals(transition.date, '2012-12-31T00:00:000Z')
+        self.assertIsNone(transition.days)
+
+    def test_parse_transition_date_deprecated(self):
+        transition = self._get_bucket_lifecycle_config()[1].transition
+        self.assertEquals(transition.date, '2012-12-31T00:00:000Z')
+        self.assertIsNone(transition.days)
+
+    def test_parse_storage_class_standard_ia(self):
+        transition = self._get_bucket_lifecycle_config()[1].transition[0]
+        self.assertEqual(transition.storage_class, 'STANDARD_IA')
+
+    def test_parse_storage_class_glacier(self):
+        transition = self._get_bucket_lifecycle_config()[0].transition[0]
         self.assertEqual(transition.storage_class, 'GLACIER')
-        self.assertEqual(response[1].transition.date, '2012-12-31T00:00:000Z')
+
+    def test_parse_storage_class_deprecated(self):
+        transition = self._get_bucket_lifecycle_config()[1].transition
+        self.assertEqual(transition.storage_class, 'STANDARD_IA')
+
+    def test_parse_multiple_lifecycle_rules(self):
+        transition = self._get_bucket_lifecycle_config()[2].transition
+        self.assertEqual(len(transition), 2)
 
     def test_expiration_with_no_transition(self):
         lifecycle = Lifecycle()
@@ -87,7 +163,14 @@ def test_expiration_is_optional(self):
             '<Transition><StorageClass>GLACIER</StorageClass><Days>30</Days>',
             xml)
 
-    def test_expiration_with_expiration_and_transition(self):
+    def test_transition_is_optional(self):
+        r = Rule('myid', 'prefix', 'Enabled')
+        xml = r.to_xml()
+        self.assertEqual(
+            '<Rule><ID>myid</ID><Prefix>prefix</Prefix><Status>Enabled</Status></Rule>',
+            xml)
+
+    def test_expiration_and_transition(self):
         t = Transition(date='2012-11-30T00:00:000Z', storage_class='GLACIER')
         r = Rule('myid', 'prefix', 'Enabled', expiration=30, transition=t)
         xml = r.to_xml()