Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'release-2.15.0'

  • Loading branch information...
commit 46d3ffea5b3576f2407bf66f3b4e9d02994e2f5f 2 parents 3bf2325 + 64c9265
Daniel G. Taylor danielgtaylor authored
31 bin/lss3
View
@@ -9,17 +9,20 @@ def sizeof_fmt(num):
num /= 1024.0
return "%3.1f %s" % (num, x)
-def list_bucket(b, prefix=None):
+def list_bucket(b, prefix=None, marker=None):
"""List everything in a bucket"""
from boto.s3.prefix import Prefix
from boto.s3.key import Key
total = 0
- query = b
+
if prefix:
if not prefix.endswith("/"):
prefix = prefix + "/"
- query = b.list(prefix=prefix, delimiter="/")
+ query = b.list(prefix=prefix, delimiter="/", marker=marker)
print "%s" % prefix
+ else:
+ query = b.list(delimiter="/", marker=marker)
+
num = 0
for k in query:
num += 1
@@ -53,25 +56,33 @@ def list_buckets(s3):
print b.name
if __name__ == "__main__":
+ import optparse
import sys
+ if len(sys.argv) < 2:
+ list_buckets(boto.connect_s3())
+ sys.exit(0)
+
+ parser = optparse.OptionParser()
+ parser.add_option('-m', '--marker',
+ help='The S3 key where the listing starts after it.')
+ options, buckets = parser.parse_args()
+ marker = options.marker
+
pairs = []
mixedCase = False
- for name in sys.argv[1:]:
+ for name in buckets:
if "/" in name:
pairs.append(name.split("/",1))
else:
pairs.append([name, None])
if pairs[-1][0].lower() != pairs[-1][0]:
mixedCase = True
-
+
if mixedCase:
s3 = boto.connect_s3(calling_format=OrdinaryCallingFormat())
else:
s3 = boto.connect_s3()
- if len(sys.argv) < 2:
- list_buckets(s3)
- else:
- for name, prefix in pairs:
- list_bucket(s3.get_bucket(name), prefix)
+ for name, prefix in pairs:
+ list_bucket(s3.get_bucket(name), prefix, marker=marker)
2  boto/__init__.py
View
@@ -36,7 +36,7 @@
import urlparse
from boto.exception import InvalidUriError
-__version__ = '2.14.0'
+__version__ = '2.15.0'
Version = __version__ # for backware compatibility
UserAgent = 'Boto/%s Python/%s %s/%s' % (
2  boto/connection.py
View
@@ -901,7 +901,7 @@ def _mexe(self, request, sender=None, override_num_retries=None,
boto.log.debug(msg)
time.sleep(next_sleep)
continue
- if response.status == 500 or response.status == 503:
+ if response.status in [500, 502, 503, 504]:
msg = 'Received %d response. ' % response.status
msg += 'Retrying in %3.1f seconds' % next_sleep
boto.log.debug(msg)
2  boto/dynamodb/item.py
View
@@ -50,11 +50,11 @@ def __init__(self, table, hash_key=None, range_key=None, attrs=None):
if range_key == None:
range_key = attrs.get(self._range_key_name, None)
self[self._range_key_name] = range_key
+ self._updates = {}
for key, value in attrs.items():
if key != self._hash_key_name and key != self._range_key_name:
self[key] = value
self.consumed_units = 0
- self._updates = {}
@property
def hash_key(self):
9 boto/ec2/connection.py
View
@@ -364,7 +364,8 @@ def deregister_image(self, image_id, delete_snapshot=False, dry_run=False):
return result
def create_image(self, instance_id, name,
- description=None, no_reboot=False, dry_run=False):
+ description=None, no_reboot=False,
+ block_device_mapping=None, dry_run=False):
"""
Will create an AMI from the instance in the running or stopped
state.
@@ -386,6 +387,10 @@ def create_image(self, instance_id, name,
responsibility of maintaining file system integrity is
left to the owner of the instance.
+ :type block_device_mapping: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping`
+ :param block_device_mapping: A BlockDeviceMapping data structure
+ describing the EBS volumes associated with the Image.
+
:type dry_run: bool
:param dry_run: Set to True if the operation should not actually run.
@@ -398,6 +403,8 @@ def create_image(self, instance_id, name,
params['Description'] = description
if no_reboot:
params['NoReboot'] = 'true'
+ if block_device_mapping:
+ block_device_mapping.ec2_build_list_params(params)
if dry_run:
params['DryRun'] = 'true'
img = self.get_object('CreateImage', params, Image, verb='POST')
28 boto/ec2/elb/__init__.py
View
@@ -188,13 +188,13 @@ def create_load_balancer(self, name, zones, listeners=None, subnets=None,
(LoadBalancerPortNumber, InstancePortNumber, Protocol, InstanceProtocol,
SSLCertificateId).
- Where;
- - LoadBalancerPortNumber and InstancePortNumber are integer
- values between 1 and 65535.
- - Protocol and InstanceProtocol is a string containing either 'TCP',
- 'SSL', 'HTTP', or 'HTTPS'
- - SSLCertificateId is the ARN of an SSL certificate loaded into
- AWS IAM
+ Where:
+ - LoadBalancerPortNumber and InstancePortNumber are integer
+ values between 1 and 65535
+ - Protocol and InstanceProtocol is a string containing either 'TCP',
+ 'SSL', 'HTTP', or 'HTTPS'
+ - SSLCertificateId is the ARN of an SSL certificate loaded into
+ AWS IAM
:rtype: :class:`boto.ec2.elb.loadbalancer.LoadBalancer`
:return: The newly created
@@ -272,13 +272,13 @@ def create_load_balancer_listeners(self, name, listeners=None, complex_listeners
(LoadBalancerPortNumber, InstancePortNumber, Protocol, InstanceProtocol,
SSLCertificateId).
- Where;
- - LoadBalancerPortNumber and InstancePortNumber are integer
- values between 1 and 65535.
- - Protocol and InstanceProtocol is a string containing either 'TCP',
- 'SSL', 'HTTP', or 'HTTPS'
- - SSLCertificateId is the ARN of an SSL certificate loaded into
- AWS IAM
+ Where:
+ - LoadBalancerPortNumber and InstancePortNumber are integer
+ values between 1 and 65535
+ - Protocol and InstanceProtocol is a string containing either 'TCP',
+ 'SSL', 'HTTP', or 'HTTPS'
+ - SSLCertificateId is the ARN of an SSL certificate loaded into
+ AWS IAM
:return: The status of the request
"""
4 boto/elastictranscoder/layer1.py
View
@@ -387,8 +387,8 @@ def create_preset(self, name=None, description=None, container=None,
:param description: A description of the preset.
:type container: string
- :param container: The container type for the output file. This value
- must be `mp4`.
+ :param container: The container type for the output file. Valid values
+ include `mp3`, `mp4`, `ogg`, `ts`, and `webm`.
:type video: dict
:param video: A section of the request body that specifies the video
3  boto/glacier/__init__.py
View
@@ -47,6 +47,9 @@ def regions():
RegionInfo(name='eu-west-1',
endpoint='glacier.eu-west-1.amazonaws.com',
connection_cls=Layer2),
+ RegionInfo(name='ap-southeast-2',
+ endpoint='glacier.ap-southeast-2.amazonaws.com',
+ connection_cls=Layer2),
]
6 boto/manage/cmdshell.py
View
@@ -34,10 +34,11 @@ class SSHClient(object):
def __init__(self, server,
host_key_file='~/.ssh/known_hosts',
- uname='root', ssh_pwd=None):
+ uname='root', timeout=None, ssh_pwd=None):
self.server = server
self.host_key_file = host_key_file
self.uname = uname
+ self._timeout = timeout
self._pkey = paramiko.RSAKey.from_private_key_file(server.ssh_key_file,
password=ssh_pwd)
self._ssh_client = paramiko.SSHClient()
@@ -52,7 +53,8 @@ def connect(self, num_retries=5):
try:
self._ssh_client.connect(self.server.hostname,
username=self.uname,
- pkey=self._pkey)
+ pkey=self._pkey,
+ timeout=self._timeout)
return
except socket.error, (value, message):
if value in (51, 61, 111):
34 boto/mws/response.py
View
@@ -59,7 +59,10 @@ def end(self, *args, **kw):
def teardown(self, *args, **kw):
if self._value is None:
- delattr(self._parent, self._name)
+ try:
+ delattr(self._parent, self._name)
+ except AttributeError: # eg. member(s) of empty MemberList(s)
+ pass
else:
setattr(self._parent, self._name, self._value)
@@ -192,7 +195,7 @@ def startElement(self, name, attrs, connection):
attribute = getattr(self, name, None)
if isinstance(attribute, DeclarativeType):
return attribute.start(name=name, attrs=attrs,
- connection=connection)
+ connection=connection)
elif attrs.getLength():
setattr(self, name, ComplexType(attrs.copy()))
else:
@@ -334,8 +337,8 @@ class ListInboundShipmentItemsByNextTokenResult(ListInboundShipmentItemsResult):
class ListInventorySupplyResult(ResponseElement):
InventorySupplyList = MemberList(
EarliestAvailability=Element(),
- SupplyDetail=MemberList(\
- EarliestAvailabileToPick=Element(),
+ SupplyDetail=MemberList(
+ EarliestAvailableToPick=Element(),
LatestAvailableToPick=Element(),
)
)
@@ -431,13 +434,13 @@ class FulfillmentPreviewItem(ResponseElement):
class FulfillmentPreview(ResponseElement):
EstimatedShippingWeight = Element(ComplexWeight)
- EstimatedFees = MemberList(\
- Element(\
+ EstimatedFees = MemberList(
+ Element(
Amount=Element(ComplexAmount),
),
)
UnfulfillablePreviewItems = MemberList(FulfillmentPreviewItem)
- FulfillmentPreviewShipments = MemberList(\
+ FulfillmentPreviewShipments = MemberList(
FulfillmentPreviewItems=MemberList(FulfillmentPreviewItem),
)
@@ -453,7 +456,8 @@ class FulfillmentOrder(ResponseElement):
class GetFulfillmentOrderResult(ResponseElement):
FulfillmentOrder = Element(FulfillmentOrder)
- FulfillmentShipment = MemberList(Element(\
+ FulfillmentShipment = MemberList(
+ Element(
FulfillmentShipmentItem=MemberList(),
FulfillmentShipmentPackage=MemberList(),
)
@@ -533,17 +537,17 @@ class Product(ResponseElement):
_namespace = 'ns2'
Identifiers = Element(MarketplaceASIN=Element(),
SKUIdentifier=Element())
- AttributeSets = Element(\
+ AttributeSets = Element(
ItemAttributes=ElementList(ItemAttributes),
)
- Relationships = Element(\
+ Relationships = Element(
VariationParent=ElementList(VariationRelationship),
)
CompetitivePricing = ElementList(CompetitivePricing)
- SalesRankings = Element(\
+ SalesRankings = Element(
SalesRank=ElementList(SalesRank),
)
- LowestOfferListings = Element(\
+ LowestOfferListings = Element(
LowestOfferListing=ElementList(LowestOfferListing),
)
@@ -611,9 +615,9 @@ class GetProductCategoriesForASINResult(GetProductCategoriesResult):
class Order(ResponseElement):
OrderTotal = Element(ComplexMoney)
ShippingAddress = Element()
- PaymentExecutionDetail = Element(\
- PaymentExecutionDetailItem=ElementList(\
- PaymentExecutionDetailItem=Element(\
+ PaymentExecutionDetail = Element(
+ PaymentExecutionDetailItem=ElementList(
+ PaymentExecutionDetailItem=Element(
Payment=Element(ComplexMoney)
)
)
6 boto/redshift/__init__.py
View
@@ -45,6 +45,12 @@ def regions():
RegionInfo(name='ap-northeast-1',
endpoint='redshift.ap-northeast-1.amazonaws.com',
connection_cls=cls),
+ RegionInfo(name='ap-southeast-1',
+ endpoint='redshift.ap-southeast-1.amazonaws.com',
+ connection_cls=cls),
+ RegionInfo(name='ap-southeast-2',
+ endpoint='redshift.ap-southeast-2.amazonaws.com',
+ connection_cls=cls),
]
32 boto/s3/bucket.py
View
@@ -63,6 +63,7 @@ class S3WebsiteEndpointTranslate:
trans_region['sa-east-1'] = 's3-website-sa-east-1'
trans_region['ap-northeast-1'] = 's3-website-ap-northeast-1'
trans_region['ap-southeast-1'] = 's3-website-ap-southeast-1'
+ trans_region['ap-southeast-2'] = 's3-website-ap-southeast-2'
@classmethod
def translate_region(self, reg):
@@ -693,7 +694,8 @@ def copy_key(self, new_key_name, src_bucket_name,
if self.name == src_bucket_name:
src_bucket = self
else:
- src_bucket = self.connection.get_bucket(src_bucket_name)
+ src_bucket = self.connection.get_bucket(
+ src_bucket_name, validate=False)
acl = src_bucket.get_xml_acl(src_key_name)
if encrypt_key:
headers[provider.server_side_encryption_header] = 'AES256'
@@ -1300,6 +1302,7 @@ def get_website_configuration(self, headers=None):
* ErrorDocument
* Key : name of object to serve when an error occurs
+
"""
return self.get_website_configuration_with_xml(headers)[0]
@@ -1320,15 +1323,24 @@ def get_website_configuration_with_xml(self, headers=None):
:rtype: 2-Tuple
:returns: 2-tuple containing:
- 1) A dictionary containing a Python representation
- of the XML response. The overall structure is:
- * WebsiteConfiguration
- * IndexDocument
- * Suffix : suffix that is appended to request that
- is for a "directory" on the website endpoint
- * ErrorDocument
- * Key : name of object to serve when an error occurs
- 2) unparsed XML describing the bucket's website configuration.
+
+ 1) A dictionary containing a Python representation \
+ of the XML response. The overall structure is:
+
+ * WebsiteConfiguration
+
+ * IndexDocument
+
+ * Suffix : suffix that is appended to request that \
+ is for a "directory" on the website endpoint
+
+ * ErrorDocument
+
+ * Key : name of object to serve when an error occurs
+
+
+ 2) unparsed XML describing the bucket's website configuration
+
"""
body = self.get_website_configuration_xml(headers=headers)
5 boto/sqs/connection.py
View
@@ -337,16 +337,19 @@ def get_all_queues(self, prefix=''):
params['QueueNamePrefix'] = prefix
return self.get_list('ListQueues', params, [('QueueUrl', Queue)])
- def get_queue(self, queue_name):
+ def get_queue(self, queue_name, owner_acct_id=None):
"""
Retrieves the queue with the given name, or ``None`` if no match
was found.
:param str queue_name: The name of the queue to retrieve.
+ :param str owner_acct_id: Optionally, the AWS account ID of the account that created the queue.
:rtype: :py:class:`boto.sqs.queue.Queue` or ``None``
:returns: The requested queue, or ``None`` if no match was found.
"""
params = {'QueueName': queue_name}
+ if owner_acct_id:
+ params['QueueOwnerAWSAccountId']=owner_acct_id
try:
return self.get_object('GetQueueUrl', params, Queue)
except SQSError:
5 docs/source/index.rst
View
@@ -54,7 +54,7 @@ Currently Supported Services
* :doc:`Cloudsearch <cloudsearch_tut>` -- (:doc:`API Reference <ref/cloudsearch>`)
* Elastic Transcoder -- (:doc:`API Reference <ref/elastictranscoder>`)
- * Simple Workflow Service (SWF) -- (:doc:`API Reference <ref/swf>`)
+ * :doc:`Simple Workflow Service (SWF) <swf_tut>` -- (:doc:`API Reference <ref/swf>`)
* :doc:`Simple Queue Service (SQS) <sqs_tut>` -- (:doc:`API Reference <ref/sqs>`)
* Simple Notification Service (SNS) -- (:doc:`API Reference <ref/sns>`)
* :doc:`Simple Email Service (SES) <ses_tut>` -- (:doc:`API Reference <ref/ses>`)
@@ -114,6 +114,8 @@ Release Notes
.. toctree::
:titlesonly:
+ releasenotes/v2.15.0
+ releasenotes/v2.14.0
releasenotes/v2.13.3
releasenotes/v2.13.2
releasenotes/v2.13.0
@@ -162,6 +164,7 @@ Release Notes
rds_tut
sqs_tut
ses_tut
+ swf_tut
cloudsearch_tut
cloudwatch_tut
vpc_tut
14 docs/source/ref/vpc.rst
View
@@ -25,6 +25,20 @@ boto.vpc.dhcpoptions
:members:
:undoc-members:
+boto.vpc.internetgateway
+------------------------
+
+.. automodule:: boto.vpc.internetgateway
+ :members:
+ :undoc-members:
+
+boto.vpc.routetable
+-------------------
+
+.. automodule:: boto.vpc.routetable
+ :members:
+ :undoc-members:
+
boto.vpc.subnet
---------------
40 docs/source/releasenotes/v2.15.0.rst
View
@@ -0,0 +1,40 @@
+boto v2.15.0
+============
+
+:date: 2013/10/17
+
+This release adds support for Amazon Elastic Transcoder audio transcoding, new
+regions for Amazon Simple Storage Service (S3), Amazon Glacier, and Amazon
+Redshift as well as new parameters in Amazon Simple Queue Service (SQS), Amazon
+Elastic Compute Cloud (EC2), and the ``lss3`` utility. Also included are
+documentation updates and fixes for S3, Amazon DynamoDB, Amazon Simple Workflow
+Service (SWF) and Amazon Marketplace Web Service (MWS).
+
+
+Features
+--------
+
+* Add SWF tutorial and code sample (:issue:`1769`, :sha:`36524f5`)
+* Add ap-southeast-2 region to S3WebsiteEndpointTranslate (:issue:`1777`,
+ :sha:`e7b0b39`)
+* Add support for ``owner_acct_id`` in SQS ``get_queue`` (:issue:`1786`,
+ :sha:`c1ad303`)
+* Add ap-southeast-2 region to Glacier (:sha:`c316266`)
+* Add ap-southeast-1 and ap-southeast-2 to Redshift (:sha:`3d67a03`)
+* Add SSH timeout option (:issue:`1755`, :sha:`d8e70ef`, :sha:`653b82b`)
+* Add support for markers in ``lss3`` (:issue:`1783`, :sha:`8ee4b1f`)
+* Add ``block_device_mapping`` to EC2 ``create_image`` (:issue:`1794`,
+ :sha:`86afe2e`)
+* Updated SWF tutorial (:issue:`1797`, :sha:`3804b16`)
+* Support Elastic Transcoder audio transcoding (:sha:`03a5087`)
+
+Bugfixes
+--------
+
+* Fix VPC module docs, ELB docs, some formatting (:issue:`1770`,
+ :sha:`75de377`)
+* Fix DynamoDB item ``attrs`` initialization (:issue:`1776`, :sha:`8454a2b`)
+* Fix parsing of empty member lists for MWS (:issue:`1785`, :sha:`7b46ca5`)
+* Fix link to release notes in docs (:sha:`a6bf794`)
+* Do not validate bucket when copying a key (:issue:`1763`, :sha:`5505113`)
+* Retry HTTP 502, 504 errors (:issue:`1798`, :sha:`c832e2d`)
433 docs/source/swf_tut.rst
View
@@ -0,0 +1,433 @@
+.. swf_tut:
+ :Authors: Slawek "oozie" Ligus <root@ooz.ie>
+
+===============================
+Amazon Simple Workflow Tutorial
+===============================
+
+This tutorial focuses on boto's interface to AWS SimpleWorkflow service.
+
+.. _SimpleWorkflow: http://aws.amazon.com/swf/
+
+What is a workflow?
+-------------------
+
+A workflow is a sequence of multiple activities aimed at accomplishing a well-defined objective. For instance, booking an airline ticket as a workflow may encompass multiple activities, such as selection of itinerary, submission of personal details, payment validation and booking confirmation.
+
+Except for the start and completion of a workflow, each step has a well-defined predecessor and successor. With that
+ - on successful completion of an activity the workflow can progress with its execution,
+ - when one of workflow's activities fails it can be retried,
+ - and when it keeps failing repeatedly the workflow may regress to the previous step to gather alternative inputs or it may simply fail at that stage.
+
+Why use workflows?
+------------------
+
+Modelling an application on a workflow provides a useful abstraction layer for writing highly-reliable programs for distributed systems, as individual responsibilities can be delegated to a set of redundant, independent and non-critical processing units.
+
+How does Amazon SWF help you accomplish this?
+---------------------------------------------
+
+Amazon SimpleWorkflow service defines an interface for workflow orchestration and provides state persistence for workflow executions.
+
+Amazon SWF applications involve communication between the following entities:
+ - The Amazon Simple Workflow Service - providing centralized orchestration and workflow state persistence,
+ - Workflow Executors - some entity starting workflow executions, typically through an action taken by a user or from a cronjob.
+ - Deciders - a program codifying the business logic, i.e. a set of instructions and decisions. Deciders take decisions based on initial set of conditions and outcomes from activities.
+ - Activity Workers - their objective is very straightforward: to take inputs, execute the tasks and return a result to the Service.
+
+The Workflow Executor contacts SWF Service and requests instantiation of a workflow. A new workflow is created and its state is stored in the service.
+The next time a decider contacts SWF service to ask for a decision task, it will be informed about a new workflow execution is taking place and it will be asked to advise SWF service on what the next steps should be. The decider then instructs the service to dispatch specific tasks to activity workers. At the next activity worker poll, the task is dispatched, then executed and the results reported back to the SWF, which then passes them onto the deciders. This exchange keeps happening repeatedly until the decider is satisfied and instructs the service to complete the execution.
+
+Prerequisites
+-------------
+
+You need a valid access and secret key. The examples below assume that you have exported them to your environment, as follows:
+
+.. code-block:: bash
+
+ bash$ export AWS_ACCESS_KEY_ID=<your access key>
+ bash$ export AWS_SECRET_ACCESS_KEY=<your secret key>
+
+Before workflows and activities can be used, they have to be registered with SWF service:
+
+.. code-block:: python
+
+ # register.py
+ import boto.swf.layer2 as swf
+ from boto.swf.exceptions import SWFTypeAlreadyExistsError, SWFDomainAlreadyExistsError
+ DOMAIN = 'boto_tutorial'
+ VERSION = '1.0'
+
+ registerables = []
+ registerables.append(swf.Domain(name=DOMAIN))
+ registerables.append(swf.WorkflowType(domain=DOMAIN, name='HelloWorkflow', version=VERSION, task_list='default'))
+ registerables.append(swf.WorkflowType(domain=DOMAIN, name='SerialWorkflow', version=VERSION, task_list='default'))
+
+ for activity_type in ('HelloWorld', 'ActivityA', 'ActivityB', 'ActivityC'):
+ registerables.append(swf.ActivityType(domain=DOMAIN, name=activity_type, version=VERSION, task_list='default'))
+
+ for swf_entity in registerables:
+ try:
+ swf_entity.register()
+ print swf_entity.name, 'registered successfully'
+ except (SWFDomainAlreadyExistsError, SWFTypeAlreadyExistsError):
+ print swf_entity.__class__.__name__, swf_entity.name, 'already exists'
+
+
+Execution of the above should produce no errors.
+
+.. code-block:: bash
+
+ bash$ python -i register.py
+ Domain boto_tutorial already exists
+ WorkflowType HelloWorkflow already exists
+ SerialWorkflow registered successfully
+ ActivityType HelloWorld already exists
+ ActivityA registered successfully
+ ActivityB registered successfully
+ ActivityC registered successfully
+ >>>
+
+HelloWorld
+----------
+
+This example is an implementation of a minimal Hello World workflow. Its execution should unfold as follows:
+
+#. A workflow execution is started.
+#. The SWF service schedules the initial decision task.
+#. A decider polls for decision tasks and receives one.
+#. The decider requests scheduling of an activity task.
+#. The SWF service schedules the greeting activity task.
+#. An activity worker polls for activity task and receives one.
+#. The worker completes the greeting activity.
+#. The SWF service schedules a decision task to inform about work outcome.
+#. The decider polls and receives a new decision task.
+#. The decider schedules workflow completion.
+#. The workflow execution finishes.
+
+Workflow logic is encoded in the decider:
+
+.. code-block:: python
+
+ # hello_decider.py
+ import boto.swf.layer2 as swf
+
+ DOMAIN = 'boto_tutorial'
+ ACTIVITY = 'HelloWorld'
+ VERSION = '1.0'
+ TASKLIST = 'default'
+
+ class HelloDecider(swf.Decider):
+
+ domain = DOMAIN
+ task_list = TASKLIST
+ version = VERSION
+
+ def run(self):
+ history = self.poll()
+ if 'events' in history:
+ # Find workflow events not related to decision scheduling.
+ workflow_events = [e for e in history['events']
+ if not e['eventType'].startswith('Decision')]
+ last_event = workflow_events[-1]
+
+ decisions = swf.Layer1Decisions()
+ if last_event['eventType'] == 'WorkflowExecutionStarted':
+ decisions.schedule_activity_task('saying_hi', ACTIVITY, VERSION, task_list=TASKLIST)
+ elif last_event['eventType'] == 'ActivityTaskCompleted':
+ decisions.complete_workflow_execution()
+ self.complete(decisions=decisions)
+ return True
+
+The activity worker is responsible for printing the greeting message when the activity task is dispatched to it by the service:
+
+.. code-block:: python
+
+ import boto.swf.layer2 as swf
+
+ DOMAIN = 'boto_tutorial'
+ VERSION = '1.0'
+ TASKLIST = 'default'
+
+ class HelloWorker(swf.ActivityWorker):
+
+ domain = DOMAIN
+ version = VERSION
+ task_list = TASKLIST
+
+ def run(self):
+ activity_task = self.poll()
+ if 'activityId' in activity_task:
+ print 'Hello, World!'
+ self.complete()
+ return True
+
+With actors implemented we can spin up a workflow execution:
+
+.. code-block:: bash
+
+ $ python
+ >>> import boto.swf.layer2 as swf
+ >>> execution = swf.WorkflowType(name='HelloWorkflow', domain='boto_tutorial', version='1.0', task_list='default').start()
+ >>>
+
+From separate terminals run an instance of a worker and a decider to carry out a workflow execution (the worker and decider may run from two independent machines).
+
+.. code-block:: bash
+
+ $ python -i hello_decider.py
+ >>> while HelloDecider().run(): pass
+ ...
+
+.. code-block:: bash
+
+ $ python -i hello_worker.py
+ >>> while HelloWorker().run(): pass
+ ...
+ Hello, World!
+
+Great. Now, to see what just happened, go back to the original terminal from which the execution was started, and read its history.
+
+.. code-block:: bash
+
+ >>> execution.history()
+ [{'eventId': 1,
+ 'eventTimestamp': 1381095173.2539999,
+ 'eventType': 'WorkflowExecutionStarted',
+ 'workflowExecutionStartedEventAttributes': {'childPolicy': 'TERMINATE',
+ 'executionStartToCloseTimeout': '3600',
+ 'parentInitiatedEventId': 0,
+ 'taskList': {'name': 'default'},
+ 'taskStartToCloseTimeout': '300',
+ 'workflowType': {'name': 'HelloWorkflow',
+ 'version': '1.0'}}},
+ {'decisionTaskScheduledEventAttributes': {'startToCloseTimeout': '300',
+ 'taskList': {'name': 'default'}},
+ 'eventId': 2,
+ 'eventTimestamp': 1381095173.2539999,
+ 'eventType': 'DecisionTaskScheduled'},
+ {'decisionTaskStartedEventAttributes': {'scheduledEventId': 2},
+ 'eventId': 3,
+ 'eventTimestamp': 1381095177.5439999,
+ 'eventType': 'DecisionTaskStarted'},
+ {'decisionTaskCompletedEventAttributes': {'scheduledEventId': 2,
+ 'startedEventId': 3},
+ 'eventId': 4,
+ 'eventTimestamp': 1381095177.855,
+ 'eventType': 'DecisionTaskCompleted'},
+ {'activityTaskScheduledEventAttributes': {'activityId': 'saying_hi',
+ 'activityType': {'name': 'HelloWorld',
+ 'version': '1.0'},
+ 'decisionTaskCompletedEventId': 4,
+ 'heartbeatTimeout': '600',
+ 'scheduleToCloseTimeout': '3900',
+ 'scheduleToStartTimeout': '300',
+ 'startToCloseTimeout': '3600',
+ 'taskList': {'name': 'default'}},
+ 'eventId': 5,
+ 'eventTimestamp': 1381095177.855,
+ 'eventType': 'ActivityTaskScheduled'},
+ {'activityTaskStartedEventAttributes': {'scheduledEventId': 5},
+ 'eventId': 6,
+ 'eventTimestamp': 1381095179.427,
+ 'eventType': 'ActivityTaskStarted'},
+ {'activityTaskCompletedEventAttributes': {'scheduledEventId': 5,
+ 'startedEventId': 6},
+ 'eventId': 7,
+ 'eventTimestamp': 1381095179.6989999,
+ 'eventType': 'ActivityTaskCompleted'},
+ {'decisionTaskScheduledEventAttributes': {'startToCloseTimeout': '300',
+ 'taskList': {'name': 'default'}},
+ 'eventId': 8,
+ 'eventTimestamp': 1381095179.6989999,
+ 'eventType': 'DecisionTaskScheduled'},
+ {'decisionTaskStartedEventAttributes': {'scheduledEventId': 8},
+ 'eventId': 9,
+ 'eventTimestamp': 1381095179.7420001,
+ 'eventType': 'DecisionTaskStarted'},
+ {'decisionTaskCompletedEventAttributes': {'scheduledEventId': 8,
+ 'startedEventId': 9},
+ 'eventId': 10,
+ 'eventTimestamp': 1381095180.026,
+ 'eventType': 'DecisionTaskCompleted'},
+ {'eventId': 11,
+ 'eventTimestamp': 1381095180.026,
+ 'eventType': 'WorkflowExecutionCompleted',
+ 'workflowExecutionCompletedEventAttributes': {'decisionTaskCompletedEventId': 10}}]
+
+
+Serial Activity Execution
+-------------------------
+
+The following example implements a basic workflow with activities executed one after another.
+
+The business logic, i.e. the serial execution of activities, is encoded in the decider:
+
+.. code-block:: python
+
+ # serial_decider.py
+ import time
+ import boto.swf.layer2 as swf
+
+ class SerialDecider(swf.Decider):
+
+ domain = 'boto_tutorial'
+ task_list = 'default_tasks'
+ version = '1.0'
+
+ def run(self):
+ history = self.poll()
+ if 'events' in history:
+ # Get a list of non-decision events to see what event came in last.
+ workflow_events = [e for e in history['events']
+ if not e['eventType'].startswith('Decision')]
+ decisions = swf.Layer1Decisions()
+ # Record latest non-decision event.
+ last_event = workflow_events[-1]
+ last_event_type = last_event['eventType']
+ if last_event_type == 'WorkflowExecutionStarted':
+ # Schedule the first activity.
+ decisions.schedule_activity_task('%s-%i' % ('ActivityA', time.time()),
+ 'ActivityA', self.version, task_list='a_tasks')
+ elif last_event_type == 'ActivityTaskCompleted':
+ # Take decision based on the name of activity that has just completed.
+ # 1) Get activity's event id.
+ last_event_attrs = last_event['activityTaskCompletedEventAttributes']
+ completed_activity_id = last_event_attrs['scheduledEventId'] - 1
+ # 2) Extract its name.
+ activity_data = history['events'][completed_activity_id]
+ activity_attrs = activity_data['activityTaskScheduledEventAttributes']
+ activity_name = activity_attrs['activityType']['name']
+ # 3) Optionally, get the result from the activity.
+ result = last_event['activityTaskCompletedEventAttributes'].get('result')
+
+ # Take the decision.
+ if activity_name == 'ActivityA':
+ decisions.schedule_activity_task('%s-%i' % ('ActivityB', time.time()),
+ 'ActivityB', self.version, task_list='b_tasks', input=result)
+ if activity_name == 'ActivityB':
+ decisions.schedule_activity_task('%s-%i' % ('ActivityC', time.time()),
+ 'ActivityC', self.version, task_list='c_tasks', input=result)
+ elif activity_name == 'ActivityC':
+ # Final activity completed. We're done.
+ decisions.complete_workflow_execution()
+
+ self.complete(decisions=decisions)
+ return True
+
+The workers only need to know which task lists to poll.
+
+.. code-block:: python
+
+ # serial_worker.py
+ import time
+ import boto.swf.layer2 as swf
+
+ class MyBaseWorker(swf.ActivityWorker):
+
+ domain = 'boto_tutorial'
+ version = '1.0'
+ task_list = None
+
+ def run(self):
+ activity_task = self.poll()
+ if 'activityId' in activity_task:
+ # Get input.
+ # Get the method for the requested activity.
+ try:
+ print 'working on activity from tasklist %s at %i' % (self.task_list, time.time())
+ self.activity(activity_task.get('input'))
+ except Exception, error:
+ self.fail(reason=str(error))
+ raise error
+
+ return True
+
+ def activity(self, activity_input):
+ raise NotImplementedError
+
+ class WorkerA(MyBaseWorker):
+ task_list = 'a_tasks'
+ def activity(self, activity_input):
+ self.complete(result="Now don't be givin him sambuca!")
+
+ class WorkerB(MyBaseWorker):
+ task_list = 'b_tasks'
+ def activity(self, activity_input):
+ self.complete()
+
+ class WorkerC(MyBaseWorker):
+ task_list = 'c_tasks'
+ def activity(self, activity_input):
+ self.complete()
+
+
+
+Spin up a workflow execution and run the decider:
+
+.. code-block:: bash
+
+ $ python
+ >>> import boto.swf.layer2 as swf
+ >>> execution = swf.WorkflowType(name='SerialWorkflow', domain='boto_tutorial', version='1.0', task_list='default_tasks').start()
+ >>>
+
+.. code-block:: bash
+
+ $ python -i serial_decider.py
+ >>> while SerialDecider().run(): pass
+ ...
+
+
+Run the workers. The activities will be executed in order:
+
+.. code-block:: bash
+
+ $ python -i serial_worker.py
+ >>> while WorkerA().run(): pass
+ ...
+ working on activity from tasklist a_tasks at 1382046291
+
+.. code-block:: bash
+
+ $ python -i serial_worker.py
+ >>> while WorkerB().run(): pass
+ ...
+ working on activity from tasklist b_tasks at 1382046541
+
+.. code-block:: bash
+
+ $ python -i serial_worker.py
+ >>> while WorkerC().run(): pass
+ ...
+ working on activity from tasklist c_tasks at 1382046560
+
+
+Looks good. Now, do the following to inspect the state and history of the execution:
+
+.. code-block:: python
+
+ >>> execution.describe()
+ {'executionConfiguration': {'childPolicy': 'TERMINATE',
+ 'executionStartToCloseTimeout': '3600',
+ 'taskList': {'name': 'default_tasks'},
+ 'taskStartToCloseTimeout': '300'},
+ 'executionInfo': {'cancelRequested': False,
+ 'closeStatus': 'COMPLETED',
+ 'closeTimestamp': 1382046560.901,
+ 'execution': {'runId': '12fQ1zSaLmI5+lLXB8ux+8U+hLOnnXNZCY9Zy+ZvXgzhE=',
+ 'workflowId': 'SerialWorkflow-1.0-1382046514'},
+ 'executionStatus': 'CLOSED',
+ 'startTimestamp': 1382046514.994,
+ 'workflowType': {'name': 'SerialWorkflow', 'version': '1.0'}},
+ 'latestActivityTaskTimestamp': 1382046560.632,
+ 'openCounts': {'openActivityTasks': 0,
+ 'openChildWorkflowExecutions': 0,
+ 'openDecisionTasks': 0,
+ 'openTimers': 0}}
+ >>> execution.history()
+ ...
+
+.. _Amazon SWF API Reference: http://docs.aws.amazon.com/amazonswf/latest/apireference/Welcome.html
+.. _StackOverflow questions: http://stackoverflow.com/questions/tagged/amazon-swf
+.. _Miscellaneous Blog Articles: http://log.ooz.ie/search/label/SimpleWorkflow
2  tests/integration/dynamodb/test_layer2.py
View
@@ -160,7 +160,7 @@ def test_layer2_basic(self):
consistent_read=True)
assert item1_copy.hash_key == item1.hash_key
assert item1_copy.range_key == item1.range_key
- for attr_name in item1_copy:
+ for attr_name in item1_attrs:
val = item1_copy[attr_name]
if isinstance(val, (int, long, float, basestring)):
assert val == item1[attr_name]
31 tests/integration/mws/test.py
View
@@ -6,6 +6,7 @@
import sys
import os
import os.path
+from datetime import datetime, timedelta
simple = os.environ.get('MWS_MERCHANT', None)
@@ -19,7 +20,7 @@
advanced = False
isolator = True
if __name__ == "__main__":
- devpath = os.path.relpath(os.path.join('..', '..'),
+ devpath = os.path.relpath(os.path.join('..', '..', '..'),
start=os.path.dirname(__file__))
sys.path = [devpath] + sys.path
advanced = simple and True or False
@@ -63,7 +64,7 @@ def test_marketplace_participations(self):
@unittest.skipUnless(simple and isolator, "skipping simple test")
def test_get_product_categories_for_asin(self):
asin = '144930544X'
- response = self.mws.get_product_categories_for_asin(\
+ response = self.mws.get_product_categories_for_asin(
MarketplaceId=self.marketplace_id,
ASIN=asin)
result = response._result
@@ -71,7 +72,7 @@ def test_get_product_categories_for_asin(self):
@unittest.skipUnless(simple and isolator, "skipping simple test")
def test_list_matching_products(self):
- response = self.mws.list_matching_products(\
+ response = self.mws.list_matching_products(
MarketplaceId=self.marketplace_id,
Query='boto')
products = response._result.Products
@@ -80,15 +81,16 @@ def test_list_matching_products(self):
@unittest.skipUnless(simple and isolator, "skipping simple test")
def test_get_matching_product(self):
asin = 'B001UDRNHO'
- response = self.mws.get_matching_product(\
+ response = self.mws.get_matching_product(
MarketplaceId=self.marketplace_id,
- ASINList=[asin,])
- product = response._result[0].Product
+ ASINList=[asin])
+ attributes = response._result[0].Product.AttributeSets.ItemAttributes
+ self.assertEqual(attributes[0].Label, 'Serengeti')
@unittest.skipUnless(simple and isolator, "skipping simple test")
def test_get_matching_product_for_id(self):
asins = ['B001UDRNHO', '144930544X']
- response = self.mws.get_matching_product_for_id(\
+ response = self.mws.get_matching_product_for_id(
MarketplaceId=self.marketplace_id,
IdType='ASIN',
IdList=asins)
@@ -99,12 +101,19 @@ def test_get_matching_product_for_id(self):
@unittest.skipUnless(simple and isolator, "skipping simple test")
def test_get_lowest_offer_listings_for_asin(self):
asin = '144930544X'
- response = self.mws.get_lowest_offer_listings_for_asin(\
+ response = self.mws.get_lowest_offer_listings_for_asin(
MarketplaceId=self.marketplace_id,
ItemCondition='New',
- ASINList=[asin,])
- product = response._result[0].Product
- self.assertTrue(product.LowestOfferListings)
+ ASINList=[asin])
+ listings = response._result[0].Product.LowestOfferListings
+ self.assertTrue(len(listings.LowestOfferListing))
+
+ @unittest.skipUnless(simple and isolator, "skipping simple test")
+ def test_list_inventory_supply(self):
+ asof = (datetime.today() - timedelta(days=30)).isoformat()
+ response = self.mws.list_inventory_supply(QueryStartDateTime=asof,
+ ResponseGroup='Basic')
+ self.assertTrue(hasattr(response._result, 'InventorySupplyList'))
if __name__ == "__main__":
unittest.main()
37 tests/unit/ec2/test_connection.py
View
@@ -9,6 +9,7 @@
import boto.ec2
from boto.regioninfo import RegionInfo
+from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
from boto.ec2.connection import EC2Connection
from boto.ec2.snapshot import Snapshot
from boto.ec2.reservedinstance import ReservedInstancesConfiguration
@@ -155,6 +156,42 @@ def test_serialized_api_args(self):
'Version'])
+class TestCreateImage(TestEC2ConnectionBase):
+ def default_body(self):
+ return """<CreateImageResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/">
+ <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId>
+ <imageId>ami-4fa54026</imageId>
+</CreateImageResponse>"""
+
+ def test_minimal(self):
+ self.set_http_response(status_code=200)
+ response = self.ec2.create_image(
+ 'instance_id', 'name')
+ self.assert_request_parameters({
+ 'Action': 'CreateImage',
+ 'InstanceId': 'instance_id',
+ 'Name': 'name'},
+ ignore_params_values=['AWSAccessKeyId', 'SignatureMethod',
+ 'SignatureVersion', 'Timestamp',
+ 'Version'])
+
+ def test_block_device_mapping(self):
+ self.set_http_response(status_code=200)
+ bdm = BlockDeviceMapping()
+ bdm['test'] = BlockDeviceType()
+ response = self.ec2.create_image(
+ 'instance_id', 'name', block_device_mapping=bdm)
+ self.assert_request_parameters({
+ 'Action': 'CreateImage',
+ 'InstanceId': 'instance_id',
+ 'Name': 'name',
+ 'BlockDeviceMapping.1.DeviceName': 'test',
+ 'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'false'},
+ ignore_params_values=['AWSAccessKeyId', 'SignatureMethod',
+ 'SignatureVersion', 'Timestamp',
+ 'Version'])
+
+
class TestCancelReservedInstancesListing(TestEC2ConnectionBase):
def default_body(self):
return """
0  tests/unit/manage/__init__.py
View
No changes.
56 tests/unit/manage/test_ssh.py
View
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+try:
+ import paramiko
+ from boto.manage.cmdshell import SSHClient
+except ImportError:
+ paramiko = None
+ SSHClient = None
+
+import mock
+
+from tests.unit import unittest
+
+
+class TestSSHTimeout(unittest.TestCase):
+ @unittest.skipIf(not paramiko, 'Paramiko missing')
+ def test_timeout(self):
+ client_tmp = paramiko.SSHClient
+
+ def client_mock():
+ client = client_tmp()
+ client.connect = mock.Mock(name='connect')
+ return client
+
+ paramiko.SSHClient = client_mock
+ paramiko.RSAKey.from_private_key_file = mock.Mock()
+
+ server = mock.Mock()
+ test = SSHClient(server)
+
+ self.assertEqual(test._ssh_client.connect.call_args[1]['timeout'], None)
+
+ test2 = SSHClient(server, timeout=30)
+
+ self.assertEqual(test2._ssh_client.connect.call_args[1]['timeout'], 30)
20 tests/unit/s3/test_bucket.py
View
@@ -1,4 +1,6 @@
# -*- coding: utf-8 -*-
+from mock import patch
+
from tests.unit import unittest
from tests.unit import AWSMockServiceTestCase
@@ -98,3 +100,21 @@ def test__get_all_query_args(self):
qa,
'initial=1&bar=%E2%98%83&max-keys=0&foo=true&some-other=thing'
)
+
+ @patch.object(Bucket, 'get_all_keys')
+ def test_bucket_copy_key_no_validate(self, mock_get_all_keys):
+ self.set_http_response(status_code=200)
+ bucket = self.service_connection.create_bucket('mybucket')
+
+ self.assertFalse(mock_get_all_keys.called)
+ self.service_connection.get_bucket('mybucket', validate=True)
+ self.assertTrue(mock_get_all_keys.called)
+
+ mock_get_all_keys.reset_mock()
+ self.assertFalse(mock_get_all_keys.called)
+ try:
+ bucket.copy_key('newkey', 'srcbucket', 'srckey', preserve_acl=True)
+ except:
+ # Will throw because of empty response.
+ pass
+ self.assertFalse(mock_get_all_keys.called)
32 tests/unit/s3/test_key.py
View
@@ -114,6 +114,38 @@ def test_400_timeout(self, sleep_mock):
self.assertTrue(k.should_retry.count, 1)
+ @mock.patch('time.sleep')
+ def test_502_bad_gateway(self, sleep_mock):
+ weird_timeout_body = "<Error><Code>BadGateway</Code></Error>"
+ self.set_http_response(status_code=502, body=weird_timeout_body)
+ b = Bucket(self.service_connection, 'mybucket')
+ k = b.new_key('test_failure')
+ fail_file = StringIO('This will pretend to be chunk-able.')
+
+ k.should_retry = counter(k.should_retry)
+ self.assertEqual(k.should_retry.count, 0)
+
+ with self.assertRaises(BotoServerError):
+ k.send_file(fail_file)
+
+ self.assertTrue(k.should_retry.count, 1)
+
+ @mock.patch('time.sleep')
+ def test_504_gateway_timeout(self, sleep_mock):
+ weird_timeout_body = "<Error><Code>GatewayTimeout</Code></Error>"
+ self.set_http_response(status_code=504, body=weird_timeout_body)
+ b = Bucket(self.service_connection, 'mybucket')
+ k = b.new_key('test_failure')
+ fail_file = StringIO('This will pretend to be chunk-able.')
+
+ k.should_retry = counter(k.should_retry)
+ self.assertEqual(k.should_retry.count, 0)
+
+ with self.assertRaises(BotoServerError):
+ k.send_file(fail_file)
+
+ self.assertTrue(k.should_retry.count, 1)
+
if __name__ == '__main__':
unittest.main()
14 tests/unit/sqs/test_connection.py
View
@@ -81,10 +81,12 @@ def test_auth_region_name_is_automatically_updated(self):
self.set_http_response(status_code=200)
self.service_connection.create_queue('my_queue')
+
# Note the region name below is 'us-west-2'.
self.assertIn('us-west-2/sqs/aws4_request',
self.actual_request.headers['Authorization'])
-
+
+
def test_set_get_auth_service_and_region_names(self):
self.service_connection.auth_service_name = 'service_name'
self.service_connection.auth_region_name = 'region_name'
@@ -93,6 +95,16 @@ def test_set_get_auth_service_and_region_names(self):
'service_name')
self.assertEqual(self.service_connection.auth_region_name, 'region_name')
+ def test_get_queue_with_owner_account_id_returns_queue(self):
+
+ self.set_http_response(status_code=200)
+ self.service_connection.create_queue('my_queue')
+
+ self.service_connection.get_queue('my_queue', '599169622985')
+
+ assert 'QueueOwnerAWSAccountId' in self.actual_request.params.keys()
+ self.assertEquals(self.actual_request.params['QueueOwnerAWSAccountId'], '599169622985')
+
if __name__ == '__main__':
unittest.main()
Please sign in to comment.
Something went wrong with that request. Please try again.