Permalink
Browse files

Merge branch 'ryansydnor-s3' into develop

* ryansydnor-s3:
  Allow s3 bucket lifecycle policies with multiple transitions
  • Loading branch information...
jamesls committed Apr 15, 2016
2 parents d1973a4 + 196cef9 commit c6d5af37cb780860f63afd6903f8876accc246b9
Showing with 213 additions and 42 deletions.
  1. +91 −16 boto/s3/lifecycle.py
  2. +29 −16 docs/source/s3_tut.rst
  3. +93 −10 tests/unit/s3/test_lifecycle.py
@@ -54,14 +54,21 @@ def __init__(self, id=None, prefix=None, status=None, expiration=None,
else:
# None or object
self.expiration = expiration
self.transition = transition
# retain backwards compatibility
if isinstance(transition, Transition):
self.transition = Transitions()
self.transition.append(transition)
elif transition:
self.transition = transition
else:
self.transition = Transitions()
def __repr__(self):
return '<Rule: %s>' % self.id
def startElement(self, name, attrs, connection):
if name == 'Transition':
self.transition = Transition()
return self.transition
elif name == 'Expiration':
self.expiration = Expiration()
@@ -139,25 +146,13 @@ class Transition(object):
in ISO 8601 format.
:ivar storage_class: The storage class to transition to. Valid
values are GLACIER.
values are GLACIER, STANDARD_IA.
"""
def __init__(self, days=None, date=None, storage_class=None):
self.days = days
self.date = date
self.storage_class = storage_class
def startElement(self, name, attrs, connection):
return None
def endElement(self, name, value, connection):
if name == 'Days':
self.days = int(value)
elif name == 'Date':
self.date = value
elif name == 'StorageClass':
self.storage_class = value
def __repr__(self):
if self.days is None:
how_long = "on: %s" % self.date
@@ -175,6 +170,86 @@ def to_xml(self):
s += '</Transition>'
return s
class Transitions(list):
"""
A container for the transitions associated with a Lifecycle's Rule configuration.
"""
def __init__(self):
self.transition_properties = 3
self.current_transition_property = 1
self.temp_days = None
self.temp_date = None
self.temp_storage_class = None
def startElement(self, name, attrs, connection):
return None
def endElement(self, name, value, connection):
if name == 'Days':
self.temp_days = int(value)
elif name == 'Date':
self.temp_date = value
elif name == 'StorageClass':
self.temp_storage_class = value
# the XML does not contain a <Transitions> tag
# but rather N number of <Transition> tags not
# structured in any sort of hierarchy.
if self.current_transition_property == self.transition_properties:
self.append(Transition(self.temp_days, self.temp_date, self.temp_storage_class))
self.temp_days = self.temp_date = self.temp_storage_class = None
self.current_transition_property = 1
else:
self.current_transition_property += 1
def to_xml(self):
"""
Returns a string containing the XML version of the Lifecycle
configuration as defined by S3.
"""
s = ''
for transition in self:
s += transition.to_xml()
return s
def add_transition(self, days=None, date=None, storage_class=None):
"""
Add a transition to this Lifecycle configuration. This only adds
the rule to the local copy. To install the new rule(s) on
the bucket, you need to pass this Lifecycle config object
to the configure_lifecycle method of the Bucket object.
:ivar days: The number of days until the object should be moved.
:ivar date: The date when the object should be moved. Should be
in ISO 8601 format.
:ivar storage_class: The storage class to transition to. Valid
values are GLACIER, STANDARD_IA.
"""
transition = Transition(days, date, storage_class)
self.append(transition)
def __first_or_default(self, prop):
for transition in self:
return getattr(transition, prop)
return None
# maintain backwards compatibility so that we can continue utilizing
# 'rule.transition.days' syntax
@property
def days(self):
return self.__first_or_default('days')
@property
def date(self):
return self.__first_or_default('date')
@property
def storage_class(self):
return self.__first_or_default('storage_class')
class Lifecycle(list):
"""
A container for the rules associated with a Lifecycle configuration.
@@ -228,7 +303,7 @@ def add_rule(self, id=None, prefix='', status='Enabled',
that are subject to the rule. The value must be a non-zero
positive integer. A Expiration object instance is also perfect.
:type transition: Transition
:type transition: Transitions
:param transition: Indicates when an object transitions to a
different storage class.
"""
@@ -442,33 +442,38 @@ And, finally, to delete all CORS configurations from a bucket::
>>> bucket.delete_cors()
Transitioning Objects to Glacier
Transitioning Objects
--------------------------------
You can configure objects in S3 to transition to Glacier after a period of
time. This is done using lifecycle policies. A lifecycle policy can also
specify that an object should be deleted after a period of time. Lifecycle
configurations are assigned to buckets and require these parameters:
S3 buckets support transitioning objects to various storage classes. This is
done using lifecycle policies. You can currently transitions objects to
Infrequent Access, Glacier, or just plain Expire. All of these options are
capable of being applied after a number of days or after a given date.
Lifecycle configurations are assigned to buckets and require these parameters:
* The object prefix that identifies the objects you are targeting.
* The object prefix that identifies the objects you are targeting. (or none)
* The action you want S3 to perform on the identified objects.
* The date (or time period) when you want S3 to perform these actions.
* The date or number of days when you want S3 to perform these actions.
For example, given a bucket ``s3-glacier-boto-demo``, we can first retrieve the
For example, given a bucket ``s3-lifecycle-boto-demo``, we can first retrieve the
bucket::
>>> import boto
>>> c = boto.connect_s3()
>>> bucket = c.get_bucket('s3-glacier-boto-demo')
>>> bucket = c.get_bucket('s3-lifecycle-boto-demo')
Then we can create a lifecycle object. In our example, we want all objects
under ``logs/*`` to transition to Glacier 30 days after the object is created.
under ``logs/*`` to transition to Standard IA 30 days after the object is created,
glacier 90 days after creation, and be deleted 120 days after creation.
::
>>> from boto.s3.lifecycle import Lifecycle, Transition, Rule
>>> to_glacier = Transition(days=30, storage_class='GLACIER')
>>> rule = Rule('ruleid', 'logs/', 'Enabled', transition=to_glacier)
>>> from boto.s3.lifecycle import Lifecycle, Transitions, Rule
>>> transitions = Transitions()
>>> transitions.add_transition(days=30, storage_class='STANDARD_IA')
>>> transitions.add_transition(days=90, storage_class='GLACIER')
>>> expiration = Expiration(days=120)
>>> rule = Rule(id='ruleid', prefix='logs/', status='Enabled', expiration=expiration, transition=transitions)
>>> lifecycle = Lifecycle()
>>> lifecycle.append(rule)
@@ -485,19 +490,27 @@ You can also retrieve the current lifecycle policy for the bucket::
>>> current = bucket.get_lifecycle_config()
>>> print current[0].transition
<Transition: in: 30 days, GLACIER>
>>> print current[0].expiration
[<Transition: in: 90 days, GLACIER>, <Transition: in: 30 days, STANDARD_IA>]
<Expiration: in: 120 days>
Note: We have deprecated directly accessing transition properties from the lifecycle
object. You must index into the transition array first.
When an object transitions to Glacier, the storage class will be
When an object transitions, the storage class will be
updated. This can be seen when you **list** the objects in a bucket::
>>> for key in bucket.list():
... print key, key.storage_class
...
<Key: s3-glacier-boto-demo,logs/testlog1.log> GLACIER
<Key: s3-lifecycle-boto-demo,logs/testlog1.log> STANDARD_IA
<Key: s3-lifecycle-boto-demo,logs/testlog2.log> GLACIER
You can also use the prefix argument to the ``bucket.list`` method::
>>> print list(b.list(prefix='logs/testlog1.log'))[0].storage_class
>>> print list(b.list(prefix='logs/testlog2.log'))[0].storage_class
u'STANDARD_IA'
u'GLACIER'
@@ -50,27 +50,103 @@ def default_body(self):
<Status>Disabled</Status>
<Transition>
<Date>2012-12-31T00:00:000Z</Date>
<StorageClass>GLACIER</StorageClass>
<StorageClass>STANDARD_IA</StorageClass>
</Transition>
<Expiration>
<Date>2012-12-31T00:00:000Z</Date>
</Expiration>
</Rule>
<Rule>
<ID>multiple-transitions</ID>
<Prefix></Prefix>
<Status>Enabled</Status>
<Transition>
<Days>30</Days>
<StorageClass>STANDARD_IA</StorageClass>
</Transition>
<Transition>
<Days>90</Days>
<StorageClass>GLACIER</StorageClass>
</Transition>
</Rule>
</LifecycleConfiguration>
"""
def test_parse_lifecycle_response(self):
def _get_bucket_lifecycle_config(self):
self.set_http_response(status_code=200)
bucket = Bucket(self.service_connection, 'mybucket')
response = bucket.get_lifecycle_config()
self.assertEqual(len(response), 2)
rule = response[0]
return bucket.get_lifecycle_config()
def test_lifecycle_response_contains_all_rules(self):
self.assertEqual(len(self._get_bucket_lifecycle_config()), 3)
def test_parse_lifecycle_id(self):
rule = self._get_bucket_lifecycle_config()[0]
self.assertEqual(rule.id, 'rule-1')
def test_parse_lifecycle_prefix(self):
rule = self._get_bucket_lifecycle_config()[0]
self.assertEqual(rule.prefix, 'prefix/foo')
def test_parse_lifecycle_no_prefix(self):
rule = self._get_bucket_lifecycle_config()[2]
self.assertEquals(rule.prefix, '')
def test_parse_lifecycle_enabled(self):
rule = self._get_bucket_lifecycle_config()[0]
self.assertEqual(rule.status, 'Enabled')
def test_parse_lifecycle_disabled(self):
rule = self._get_bucket_lifecycle_config()[1]
self.assertEqual(rule.status, 'Disabled')
def test_parse_expiration_days(self):
rule = self._get_bucket_lifecycle_config()[0]
self.assertEqual(rule.expiration.days, 365)
self.assertIsNone(rule.expiration.date)
transition = rule.transition
self.assertEqual(transition.days, 30)
def test_parse_expiration_date(self):
rule = self._get_bucket_lifecycle_config()[1]
self.assertEqual(rule.expiration.date, '2012-12-31T00:00:000Z')
def test_parse_expiration_not_required(self):
rule = self._get_bucket_lifecycle_config()[2]
self.assertIsNone(rule.expiration)
def test_parse_transition_days(self):
transition = self._get_bucket_lifecycle_config()[0].transition[0]
self.assertEquals(transition.days, 30)
self.assertIsNone(transition.date)
def test_parse_transition_days_deprecated(self):
transition = self._get_bucket_lifecycle_config()[0].transition
self.assertEquals(transition.days, 30)
self.assertIsNone(transition.date)
def test_parse_transition_date(self):
transition = self._get_bucket_lifecycle_config()[1].transition[0]
self.assertEquals(transition.date, '2012-12-31T00:00:000Z')
self.assertIsNone(transition.days)
def test_parse_transition_date_deprecated(self):
transition = self._get_bucket_lifecycle_config()[1].transition
self.assertEquals(transition.date, '2012-12-31T00:00:000Z')
self.assertIsNone(transition.days)
def test_parse_storage_class_standard_ia(self):
transition = self._get_bucket_lifecycle_config()[1].transition[0]
self.assertEqual(transition.storage_class, 'STANDARD_IA')
def test_parse_storage_class_glacier(self):
transition = self._get_bucket_lifecycle_config()[0].transition[0]
self.assertEqual(transition.storage_class, 'GLACIER')
self.assertEqual(response[1].transition.date, '2012-12-31T00:00:000Z')
def test_parse_storage_class_deprecated(self):
transition = self._get_bucket_lifecycle_config()[1].transition
self.assertEqual(transition.storage_class, 'STANDARD_IA')
def test_parse_multiple_lifecycle_rules(self):
transition = self._get_bucket_lifecycle_config()[2].transition
self.assertEqual(len(transition), 2)
def test_expiration_with_no_transition(self):
lifecycle = Lifecycle()
@@ -87,7 +163,14 @@ def test_expiration_is_optional(self):
'<Transition><StorageClass>GLACIER</StorageClass><Days>30</Days>',
xml)
def test_expiration_with_expiration_and_transition(self):
def test_transition_is_optional(self):
r = Rule('myid', 'prefix', 'Enabled')
xml = r.to_xml()
self.assertEqual(
'<Rule><ID>myid</ID><Prefix>prefix</Prefix><Status>Enabled</Status></Rule>',
xml)
def test_expiration_and_transition(self):
t = Transition(date='2012-11-30T00:00:000Z', storage_class='GLACIER')
r = Rule('myid', 'prefix', 'Enabled', expiration=30, transition=t)
xml = r.to_xml()

0 comments on commit c6d5af3

Please sign in to comment.