Permalink
Browse files

Merge remote-tracking branch 'upstream/develop' into consistent-ident…

…ity-comparison-none

Conflicts:
	boto/manage/server.py
  • Loading branch information...
2 parents 2e7e84b + cf693ff commit 6744439d15bb2e6e0ee5152f0c0df665c910a8e2 @oxtopus oxtopus committed Dec 27, 2013
Showing with 2,136 additions and 623 deletions.
  1. +2 −2 README.rst
  2. +4 −4 bin/s3put
  3. +5 −1 boto/__init__.py
  4. +198 −13 boto/auth.py
  5. +2 −2 boto/beanstalk/layer1.py
  6. +1 −0 boto/cloudformation/__init__.py
  7. +1 −1 boto/cloudformation/connection.py
  8. +1 −1 boto/cloudfront/__init__.py
  9. +5 −4 boto/cloudfront/distribution.py
  10. +4 −4 boto/cloudfront/object.py
  11. +162 −113 boto/cloudtrail/layer1.py
  12. +2 −2 boto/connection.py
  13. +2 −2 boto/contrib/ymlmessage.py
  14. +1 −1 boto/datapipeline/layer1.py
  15. +1 −1 boto/directconnect/layer1.py
  16. +3 −0 boto/dynamodb/__init__.py
  17. +1 −1 boto/dynamodb/layer1.py
  18. +3 −0 boto/dynamodb2/__init__.py
  19. +1 −1 boto/dynamodb2/layer1.py
  20. +30 −7 boto/dynamodb2/results.py
  21. +30 −5 boto/dynamodb2/table.py
  22. +1 −0 boto/ec2/__init__.py
  23. +1 −1 boto/ec2/address.py
  24. +2 −1 boto/ec2/autoscale/__init__.py
  25. +5 −5 boto/ec2/bundleinstance.py
  26. +2 −1 boto/ec2/cloudwatch/__init__.py
  27. +24 −4 boto/ec2/connection.py
  28. +1 −1 boto/ec2/ec2object.py
  29. +2 −1 boto/ec2/elb/__init__.py
  30. +10 −2 boto/ec2/image.py
  31. +3 −3 boto/ec2/instance.py
  32. +1 −1 boto/ec2/keypair.py
  33. +1 −1 boto/ec2/launchspecification.py
  34. +3 −2 boto/ec2/networkinterface.py
  35. +1 −1 boto/ec2/placementgroup.py
  36. +3 −3 boto/ec2/regioninfo.py
  37. +6 −5 boto/ec2/reservedinstance.py
  38. +3 −2 boto/ec2/securitygroup.py
  39. +1 −1 boto/ec2/snapshot.py
  40. +1 −1 boto/ec2/spotdatafeedsubscription.py
  41. +3 −2 boto/ec2/spotinstancerequest.py
  42. +3 −3 boto/ec2/spotpricehistory.py
  43. +1 −1 boto/ec2/vmtype.py
  44. +2 −2 boto/ec2/volume.py
  45. +5 −5 boto/ec2/zone.py
  46. +5 −5 boto/ecs/__init__.py
  47. +6 −6 boto/ecs/item.py
  48. +3 −0 boto/elasticache/__init__.py
  49. +2 −2 boto/elasticache/layer1.py
  50. +41 −11 boto/elastictranscoder/layer1.py
  51. +3 −0 boto/emr/__init__.py
  52. +1 −1 boto/emr/connection.py
  53. +5 −5 boto/emr/step.py
  54. +24 −22 boto/exception.py
  55. +1 −1 boto/fps/connection.py
  56. +25 −17 boto/fps/response.py
  57. +3 −0 boto/glacier/__init__.py
  58. +2 −2 boto/glacier/layer1.py
  59. +6 −6 boto/gs/connection.py
  60. +3 −0 boto/iam/__init__.py
  61. +1 −2 boto/iam/connection.py
  62. +1 −1 boto/kinesis/layer1.py
  63. +25 −25 boto/manage/server.py
  64. +11 −11 boto/manage/task.py
  65. +7 −7 boto/manage/volume.py
  66. +5 −5 boto/mashups/server.py
  67. +6 −6 boto/mturk/connection.py
  68. +8 −8 boto/mturk/qualification.py
  69. +1 −1 boto/mturk/question.py
  70. +1 −1 boto/mws/connection.py
  71. +13 −13 boto/mws/response.py
  72. +2 −2 boto/opsworks/layer1.py
  73. +2 −2 boto/pyami/bootstrap.py
  74. +2 −0 boto/pyami/config.py
  75. +4 −4 boto/pyami/copybot.py
  76. +7 −7 boto/pyami/installers/ubuntu/ebs.py
  77. +12 −10 boto/rds/__init__.py
  78. +2 −2 boto/rds/regioninfo.py
  79. +1 −1 boto/redshift/layer1.py
  80. +13 −13 boto/roboto/awsqueryrequest.py
  81. +4 −4 boto/roboto/awsqueryservice.py
  82. +7 −7 boto/roboto/param.py
  83. +2 −2 boto/route53/connection.py
  84. +3 −3 boto/route53/record.py
  85. +3 −0 boto/s3/__init__.py
  86. +108 −13 boto/s3/bucket.py
  87. +23 −12 boto/s3/bucketlistresultset.py
  88. +6 −3 boto/s3/connection.py
  89. +12 −5 boto/s3/key.py
  90. +12 −1 boto/s3/multipart.py
  91. +2 −2 boto/sdb/connection.py
  92. +31 −31 boto/sdb/db/property.py
  93. +1 −1 boto/sdb/domain.py
  94. +2 −2 boto/sdb/regioninfo.py
  95. +3 −3 boto/services/service.py
  96. +9 −9 boto/services/servicedef.py
  97. +3 −3 boto/services/sonofmmm.py
  98. +1 −1 boto/ses/connection.py
  99. +3 −0 boto/sns/__init__.py
  100. +2 −2 boto/sns/connection.py
  101. +3 −1 boto/sqs/__init__.py
  102. +119 −0 boto/sqs/bigmessage.py
  103. +1 −1 boto/sqs/connection.py
  104. +3 −3 boto/sqs/message.py
  105. +2 −2 boto/sqs/queue.py
  106. +2 −2 boto/sqs/regioninfo.py
  107. +4 −2 boto/sts/__init__.py
  108. +5 −3 boto/sts/connection.py
  109. +137 −67 boto/support/layer1.py
  110. +1 −0 boto/swf/__init__.py
  111. +1 −1 boto/swf/layer1.py
  112. +1 −1 boto/utils.py
  113. +1 −1 boto/vpc/__init__.py
  114. +1 −1 boto/vpc/customergateway.py
  115. +6 −6 boto/vpc/dhcpoptions.py
  116. +1 −1 boto/vpc/internetgateway.py
  117. +1 −1 boto/vpc/networkacl.py
  118. +1 −1 boto/vpc/routetable.py
  119. +1 −1 boto/vpc/subnet.py
  120. +1 −1 boto/vpc/vpc.py
  121. +1 −1 boto/vpc/vpnconnection.py
  122. +2 −2 boto/vpc/vpngateway.py
  123. +62 −0 docs/source/dynamodb2_tut.rst
  124. +3 −0 docs/source/index.rst
  125. +43 −0 docs/source/releasenotes/v2.21.0.rst
  126. +21 −0 docs/source/releasenotes/v2.21.1.rst
  127. +13 −0 docs/source/releasenotes/v2.21.2.rst
  128. +74 −0 scripts/git-release-notes.py
  129. +50 −0 tests/integration/dynamodb2/forum_test_data.json
  130. +51 −0 tests/integration/dynamodb2/test_highlevel.py
  131. +80 −0 tests/integration/sqs/test_bigmessage.py
  132. +204 −0 tests/unit/auth/test_sigv4.py
  133. +3 −3 tests/unit/dynamodb2/test_table.py
  134. +42 −0 tests/unit/ec2/test_connection.py
  135. +0 −1 tests/unit/ec2/test_volume.py
  136. +2 −2 tests/unit/elasticache/test_api_interface.py
  137. +51 −0 tests/unit/s3/test_bucket.py
  138. +50 −0 tests/unit/s3/test_connection.py
  139. +28 −0 tests/unit/sqs/test_message.py
  140. +12 −0 tests/unit/sts/test_connection.py
  141. +17 −1 tests/unit/utils/test_utils.py
View
@@ -1,9 +1,9 @@
####
boto
####
-boto 2.20.1
+boto 2.21.2
-Released: 13-December-2013
+Released: 24-December-2013
.. image:: https://travis-ci.org/boto/boto.png?branch=develop
:target: https://travis-ci.org/boto/boto
View
@@ -298,18 +298,18 @@ def main():
aws_secret_access_key = a
if o in ('-r', '--reduced'):
reduced = True
- if o in ('--header'):
+ if o == '--header':
(k, v) = a.split("=", 1)
headers[k] = v
- if o in ('--host'):
+ if o == '--host':
host = a
- if o in ('--multipart'):
+ if o == '--multipart':
if multipart_capable:
multipart_requested = True
else:
print "multipart upload requested but not capable"
sys.exit(4)
- if o in ('--region'):
+ if o == '--region':
regions = boto.s3.regions()
for region_info in regions:
if region_info.name == a:
View
@@ -27,6 +27,7 @@
from boto.pyami.config import Config, BotoConfigLocations
from boto.storage_uri import BucketStorageUri, FileStorageUri
import boto.plugin
+import datetime
import os
import platform
import re
@@ -36,9 +37,12 @@
import urlparse
from boto.exception import InvalidUriError
-__version__ = '2.20.1'
+__version__ = '2.21.2'
Version = __version__ # for backware compatibility
+# http://bugs.python.org/issue7980
+datetime.datetime.strptime('', '')
+
UserAgent = 'Boto/%s Python/%s %s/%s' % (
__version__,
platform.python_version(),
View
@@ -39,6 +39,7 @@
import sys
import time
import urllib
+import urlparse
import posixpath
from boto.auth_handler import AuthHandler
@@ -108,7 +109,7 @@ class AnonAuthHandler(AuthHandler, HmacKeys):
capability = ['anon']
def __init__(self, host, config, provider):
- AuthHandler.__init__(self, host, config, provider)
+ super(AnonAuthHandler, self).__init__(host, config, provider)
def add_auth(self, http_request, **kwargs):
pass
@@ -352,10 +353,15 @@ def canonical_headers(self, headers_to_sign):
case, sorting them in alphabetical order and then joining
them into a string, separated by newlines.
"""
- l = sorted(['%s:%s' % (n.lower().strip(),
- ' '.join(headers_to_sign[n].strip().split()))
- for n in headers_to_sign])
- return '\n'.join(l)
+ canonical = []
+
+ for header in headers_to_sign:
+ c_name = header.lower().strip()
+ raw_value = headers_to_sign[header]
+ c_value = ' '.join(raw_value.strip().split())
+ canonical.append('%s:%s' % (c_name, c_value))
+
+ return '\n'.join(sorted(canonical))
def signed_headers(self, headers_to_sign):
l = ['%s' % n.lower().strip() for n in headers_to_sign]
@@ -400,14 +406,11 @@ def scope(self, http_request):
scope.append('aws4_request')
return '/'.join(scope)
- def credential_scope(self, http_request):
- scope = []
- http_request.timestamp = http_request.headers['X-Amz-Date'][0:8]
- scope.append(http_request.timestamp)
- # The service_name and region_name either come from:
- # * The service_name/region_name attrs or (if these values are None)
- # * parsed from the endpoint <service>.<region>.amazonaws.com.
- parts = http_request.host.split('.')
+ def split_host_parts(self, host):
+ return host.split('.')
+
+ def determine_region_name(self, host):
+ parts = self.split_host_parts(host)
if self.region_name is not None:
region_name = self.region_name
elif len(parts) > 1:
@@ -421,11 +424,25 @@ def credential_scope(self, http_request):
else:
region_name = parts[0]
+ return region_name
+
+ def determine_service_name(self, host):
+ parts = self.split_host_parts(host)
if self.service_name is not None:
service_name = self.service_name
else:
service_name = parts[0]
+ return service_name
+ def credential_scope(self, http_request):
+ scope = []
+ http_request.timestamp = http_request.headers['X-Amz-Date'][0:8]
+ scope.append(http_request.timestamp)
+ # The service_name and region_name either come from:
+ # * The service_name/region_name attrs or (if these values are None)
+ # * parsed from the endpoint <service>.<region>.amazonaws.com.
+ region_name = self.determine_region_name(http_request.host)
+ service_name = self.determine_service_name(http_request.host)
http_request.service_name = service_name
http_request.region_name = region_name
@@ -495,6 +512,153 @@ def add_auth(self, req, **kwargs):
req.headers['Authorization'] = ','.join(l)
+class S3HmacAuthV4Handler(HmacAuthV4Handler, AuthHandler):
+ """
+ Implements a variant of Version 4 HMAC authorization specific to S3.
+ """
+ capability = ['hmac-v4-s3']
+
+ def __init__(self, *args, **kwargs):
+ super(S3HmacAuthV4Handler, self).__init__(*args, **kwargs)
+
+ if self.region_name:
+ self.region_name = self.clean_region_name(self.region_name)
+
+ def clean_region_name(self, region_name):
+ if region_name.startswith('s3-'):
+ return region_name[3:]
+
+ return region_name
+
+ def canonical_uri(self, http_request):
+ # S3 does **NOT** do path normalization that SigV4 typically does.
+ # Urlencode the path, **NOT** ``auth_path`` (because vhosting).
+ path = urlparse.urlparse(http_request.path)
+ encoded = urllib.quote(path.path)
+ return encoded
+
+ def host_header(self, host, http_request):
+ port = http_request.port
+ secure = http_request.protocol == 'https'
+ if ((port == 80 and not secure) or (port == 443 and secure)):
+ return http_request.host
+ return '%s:%s' % (http_request.host, port)
+
+ def headers_to_sign(self, http_request):
+ """
+ Select the headers from the request that need to be included
+ in the StringToSign.
+ """
+ host_header_value = self.host_header(self.host, http_request)
+ headers_to_sign = {}
+ headers_to_sign = {'Host': host_header_value}
+ for name, value in http_request.headers.items():
+ lname = name.lower()
+ # Hooray for the only difference! The main SigV4 signer only does
+ # ``Host`` + ``x-amz-*``. But S3 wants pretty much everything
+ # signed, except for authorization itself.
+ if not lname in ['authorization']:
+ headers_to_sign[name] = value
+ return headers_to_sign
+
+ def determine_region_name(self, host):
+ # S3's different format(s) of representing region/service from the
+ # rest of AWS makes this hurt too.
+ #
+ # Possible domain formats:
+ # - s3.amazonaws.com (Classic)
+ # - s3-us-west-2.amazonaws.com (Specific region)
+ # - bukkit.s3.amazonaws.com (Vhosted Classic)
+ # - bukkit.s3-ap-northeast-1.amazonaws.com (Vhosted specific region)
+ # - s3.cn-north-1.amazonaws.com.cn - (Bejing region)
+ # - bukkit.s3.cn-north-1.amazonaws.com.cn - (Vhosted Bejing region)
+ parts = self.split_host_parts(host)
+
+ if self.region_name is not None:
+ region_name = self.region_name
+ else:
+ # Classic URLs - s3-us-west-2.amazonaws.com
+ if len(parts) == 3:
+ region_name = self.clean_region_name(parts[0])
+
+ # Special-case for Classic.
+ if region_name == 's3':
+ region_name = 'us-east-1'
+ else:
+ # Iterate over the parts in reverse order.
+ for offset, part in enumerate(reversed(parts)):
+ part = part.lower()
+
+ # Look for the first thing starting with 's3'.
+ # Until there's a ``.s3`` TLD, we should be OK. :P
+ if part == 's3':
+ # If it's by itself, the region is the previous part.
+ region_name = parts[-offset]
+ break
+ elif part.startswith('s3-'):
+ region_name = self.clean_region_name(part)
+ break
+
+ return region_name
+
+ def determine_service_name(self, host):
+ # Should this signing mechanism ever be used for anything else, this
+ # will fail. Consider utilizing the logic from the parent class should
+ # you find yourself here.
+ return 's3'
+
+ def mangle_path_and_params(self, req):
+ """
+ Returns a copy of the request object with fixed ``auth_path/params``
+ attributes from the original.
+ """
+ modified_req = copy.copy(req)
+
+ # Unlike the most other services, in S3, ``req.params`` isn't the only
+ # source of query string parameters.
+ # Because of the ``query_args``, we may already have a query string
+ # **ON** the ``path/auth_path``.
+ # Rip them apart, so the ``auth_path/params`` can be signed
+ # appropriately.
+ parsed_path = urlparse.urlparse(modified_req.auth_path)
+ modified_req.auth_path = parsed_path.path
+
+ if modified_req.params is None:
+ modified_req.params = {}
+
+ raw_qs = parsed_path.query
+ existing_qs = urlparse.parse_qs(
+ raw_qs,
+ keep_blank_values=True
+ )
+
+ # ``parse_qs`` will return lists. Don't do that unless there's a real,
+ # live list provided.
+ for key, value in existing_qs.items():
+ if isinstance(value, (list, tuple)):
+ if len(value) == 1:
+ existing_qs[key] = value[0]
+
+ modified_req.params.update(existing_qs)
+ return modified_req
+
+ def payload(self, http_request):
+ if http_request.headers.get('x-amz-content-sha256'):
+ return http_request.headers['x-amz-content-sha256']
+
+ return super(S3HmacAuthV4Handler, self).payload(http_request)
+
+ def add_auth(self, req, **kwargs):
+ if not 'x-amz-content-sha256' in req.headers:
+ if '_sha256' in req.headers:
+ req.headers['x-amz-content-sha256'] = req.headers.pop('_sha256')
+ else:
+ req.headers['x-amz-content-sha256'] = self.payload(req)
+
+ req = self.mangle_path_and_params(req)
+ return super(S3HmacAuthV4Handler, self).add_auth(req, **kwargs)
+
+
class QueryAuthHandler(AuthHandler):
"""
Provides pure query construction (no actual signing).
@@ -721,3 +885,24 @@ def get_auth_handler(host, config, provider, requested_capability=None):
# user could override this with a .boto config that includes user-specific
# credentials (for access to user data).
return ready_handlers[-1]
+
+
+def detect_potential_sigv4(func):
+ def _wrapper(self):
+ if hasattr(self, 'region'):
+ if getattr(self.region, 'endpoint', ''):
+ if '.cn-' in self.region.endpoint:
+ return ['hmac-v4']
+
+ return func(self)
+ return _wrapper
+
+
+def detect_potential_s3sigv4(func):
+ def _wrapper(self):
+ if hasattr(self, 'host'):
+ if '.cn-' in self.host:
+ return ['hmac-v4-s3']
+
+ return func(self)
+ return _wrapper
View
@@ -45,7 +45,7 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
region = RegionInfo(self, self.DefaultRegionName,
self.DefaultRegionEndpoint)
self.region = region
- AWSQueryConnection.__init__(self, aws_access_key_id,
+ super(Layer1, self).__init__(aws_access_key_id,
aws_secret_access_key,
is_secure, port, proxy, proxy_port,
proxy_user, proxy_pass,
@@ -1116,7 +1116,7 @@ def update_environment(self, environment_id=None, environment_name=None,
:type tier_version: string
:type tier_version: The version of the tier. Valid values
currently are "1.0". Defaults to "1.0".
-
+
:raises: InsufficientPrivilegesException
"""
params = {}
@@ -32,6 +32,7 @@
'ap-northeast-1': 'cloudformation.ap-northeast-1.amazonaws.com',
'ap-southeast-1': 'cloudformation.ap-southeast-1.amazonaws.com',
'ap-southeast-2': 'cloudformation.ap-southeast-2.amazonaws.com',
+ 'cn-north-1': 'cloudformation.cn-north-1.amazonaws.com.cn',
}
@@ -57,7 +57,7 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
region = RegionInfo(self, self.DefaultRegionName,
self.DefaultRegionEndpoint, CloudFormationConnection)
self.region = region
- AWSQueryConnection.__init__(self, aws_access_key_id,
+ super(CloudFormationConnection, self).__init__(aws_access_key_id,
aws_secret_access_key,
is_secure, port, proxy, proxy_port,
proxy_user, proxy_pass,
@@ -44,7 +44,7 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
port=None, proxy=None, proxy_port=None,
host=DefaultHost, debug=0, security_token=None,
validate_certs=True):
- AWSAuthConnection.__init__(self, host,
+ super(CloudFrontConnection, self).__init__(host,
aws_access_key_id, aws_secret_access_key,
True, port, proxy, proxy_port, debug=debug,
security_token=security_token,
@@ -176,7 +176,7 @@ class StreamingDistributionConfig(DistributionConfig):
def __init__(self, connection=None, origin='', enabled=False,
caller_reference='', cnames=None, comment='',
trusted_signers=None, logging=None):
- DistributionConfig.__init__(self, connection=connection,
+ super(StreamingDistributionConfig, self).__init__(connection=connection,
origin=origin, enabled=enabled,
caller_reference=caller_reference,
cnames=cnames, comment=comment,
@@ -684,16 +684,17 @@ class StreamingDistribution(Distribution):
def __init__(self, connection=None, config=None, domain_name='',
id='', last_modified_time=None, status=''):
- Distribution.__init__(self, connection, config, domain_name,
- id, last_modified_time, status)
+ super(StreamingDistribution, self).__init__(connection, config,
+ domain_name, id, last_modified_time, status)
self._object_class = StreamingObject
def startElement(self, name, attrs, connection):
if name == 'StreamingDistributionConfig':
self.config = StreamingDistributionConfig()
return self.config
else:
- return Distribution.startElement(self, name, attrs, connection)
+ return super(StreamingDistribution, self).startElement(name, attrs,
+ connection)
def update(self, enabled=None, cnames=None, comment=None):
"""
Oops, something went wrong.

0 comments on commit 6744439

Please sign in to comment.