Permalink
Browse files

Merge branch 'release-2.21.0'

  • Loading branch information...
danielgtaylor committed Dec 20, 2013
2 parents b98f85c + d24ca8c commit 87a45cd2b7b5557e22722da15f22c915ce2bec66
Showing with 2,719 additions and 516 deletions.
  1. +2 −2 README.rst
  2. +1 −1 boto/__init__.py
  3. +198 −34 boto/auth.py
  4. +53 −5 boto/beanstalk/layer1.py
  5. +2 −2 boto/cloudformation/template.py
  6. +8 −9 boto/cloudfront/identity.py
  7. +6 −7 boto/cloudfront/signers.py
  8. +161 −112 boto/cloudtrail/layer1.py
  9. +126 −4 boto/dynamodb2/fields.py
  10. +97 −29 boto/dynamodb2/table.py
  11. +1 −1 boto/ec2/autoscale/__init__.py
  12. +6 −2 boto/ec2/connection.py
  13. +3 −4 boto/ec2/group.py
  14. +3 −3 boto/ec2/image.py
  15. +1 −4 boto/ec2/instance.py
  16. +2 −3 boto/ec2/snapshot.py
  17. +1 −3 boto/ec2/volume.py
  18. +1 −1 boto/elasticache/layer1.py
  19. +38 −8 boto/elastictranscoder/layer1.py
  20. +57 −0 boto/emr/connection.py
  21. +4 −0 boto/emr/emrobject.py
  22. +20 −6 boto/exception.py
  23. +1 −1 boto/mturk/connection.py
  24. +3 −3 boto/mturk/layoutparam.py
  25. +2 −2 boto/mturk/notification.py
  26. +2 −2 boto/mturk/price.py
  27. +5 −5 boto/mturk/qualification.py
  28. +3 −3 boto/mturk/question.py
  29. +488 −61 boto/opsworks/layer1.py
  30. +2 −2 boto/pyami/scriptbase.py
  31. +24 −6 boto/rds/__init__.py
  32. +7 −7 boto/s3/acl.py
  33. +108 −14 boto/s3/bucket.py
  34. +30 −19 boto/s3/bucketlistresultset.py
  35. +3 −3 boto/s3/bucketlogging.py
  36. +3 −1 boto/s3/connection.py
  37. +2 −2 boto/s3/deletemarker.py
  38. +12 −5 boto/s3/key.py
  39. +12 −1 boto/s3/multipart.py
  40. +3 −3 boto/s3/user.py
  41. +9 −9 boto/sdb/db/manager/xmlmanager.py
  42. +18 −18 boto/sdb/domain.py
  43. +4 −4 boto/sdb/queryresultset.py
  44. +4 −4 boto/services/result.py
  45. +2 −2 boto/services/submit.py
  46. +119 −0 boto/sqs/bigmessage.py
  47. +9 −9 boto/sqs/message.py
  48. +3 −3 boto/sqs/queue.py
  49. +136 −66 boto/support/layer1.py
  50. +1 −1 boto/swf/layer1_decisions.py
  51. +2 −0 docs/source/index.rst
  52. +1 −0 docs/source/ref/index.rst
  53. +26 −0 docs/source/ref/kinesis.rst
  54. +32 −0 docs/source/releasenotes/v2.21.0.rst
  55. +38 −1 tests/integration/dynamodb2/test_highlevel.py
  56. +9 −9 tests/integration/ec2/autoscale/test_connection.py
  57. +2 −2 tests/integration/opsworks/test_layer1.py
  58. +80 −0 tests/integration/sqs/test_bigmessage.py
  59. +204 −0 tests/unit/auth/test_sigv4.py
  60. +30 −0 tests/unit/beanstalk/test_layer1.py
  61. +215 −6 tests/unit/dynamodb2/test_table.py
  62. +24 −0 tests/unit/ec2/test_connection.py
  63. +2 −2 tests/unit/elasticache/test_api_interface.py
  64. +119 −0 tests/unit/emr/test_connection.py
  65. +51 −0 tests/unit/s3/test_bucket.py
  66. +50 −0 tests/unit/s3/test_connection.py
  67. +28 −0 tests/unit/sqs/test_message.py
View
@@ -1,9 +1,9 @@
####
boto
####
-boto 2.20.1
+boto 2.21.0
-Released: 13-December-2013
+Released: 19-December-2013
.. image:: https://travis-ci.org/boto/boto.png?branch=develop
:target: https://travis-ci.org/boto/boto
View
@@ -36,7 +36,7 @@
import urlparse
from boto.exception import InvalidUriError
-__version__ = '2.20.1'
+__version__ = '2.21.0'
Version = __version__ # for backware compatibility
UserAgent = 'Boto/%s Python/%s %s/%s' % (
View
@@ -39,35 +39,15 @@
import sys
import time
import urllib
+import urlparse
import posixpath
from boto.auth_handler import AuthHandler
from boto.exception import BotoClientError
-#
-# the following is necessary because of the incompatibilities
-# between Python 2.4, 2.5, and 2.6 as well as the fact that some
-# people running 2.4 have installed hashlib as a separate module
-# this fix was provided by boto user mccormix.
-# see: http://code.google.com/p/boto/issues/detail?id=172
-# for more details.
-#
+
try:
from hashlib import sha1 as sha
from hashlib import sha256 as sha256
-
- if sys.version[:3] == "2.4":
- # we are using an hmac that expects a .new() method.
- class Faker:
- def __init__(self, which):
- self.which = which
- self.digest_size = self.which().digest_size
-
- def new(self, *args, **kwargs):
- return self.which(*args, **kwargs)
-
- sha = Faker(sha)
- sha256 = Faker(sha256)
-
except ImportError:
import sha
sha256 = None
@@ -373,10 +353,15 @@ def canonical_headers(self, headers_to_sign):
case, sorting them in alphabetical order and then joining
them into a string, separated by newlines.
"""
- l = sorted(['%s:%s' % (n.lower().strip(),
- ' '.join(headers_to_sign[n].strip().split()))
- for n in headers_to_sign])
- return '\n'.join(l)
+ canonical = []
+
+ for header in headers_to_sign:
+ c_name = header.lower().strip()
+ raw_value = headers_to_sign[header]
+ c_value = ' '.join(raw_value.strip().split())
+ canonical.append('%s:%s' % (c_name, c_value))
+
+ return '\n'.join(sorted(canonical))
def signed_headers(self, headers_to_sign):
l = ['%s' % n.lower().strip() for n in headers_to_sign]
@@ -421,14 +406,11 @@ def scope(self, http_request):
scope.append('aws4_request')
return '/'.join(scope)
- def credential_scope(self, http_request):
- scope = []
- http_request.timestamp = http_request.headers['X-Amz-Date'][0:8]
- scope.append(http_request.timestamp)
- # The service_name and region_name either come from:
- # * The service_name/region_name attrs or (if these values are None)
- # * parsed from the endpoint <service>.<region>.amazonaws.com.
- parts = http_request.host.split('.')
+ def split_host_parts(self, host):
+ return host.split('.')
+
+ def determine_region_name(self, host):
+ parts = self.split_host_parts(host)
if self.region_name is not None:
region_name = self.region_name
elif len(parts) > 1:
@@ -442,11 +424,25 @@ def credential_scope(self, http_request):
else:
region_name = parts[0]
+ return region_name
+
+ def determine_service_name(self, host):
+ parts = self.split_host_parts(host)
if self.service_name is not None:
service_name = self.service_name
else:
service_name = parts[0]
+ return service_name
+ def credential_scope(self, http_request):
+ scope = []
+ http_request.timestamp = http_request.headers['X-Amz-Date'][0:8]
+ scope.append(http_request.timestamp)
+ # The service_name and region_name either come from:
+ # * The service_name/region_name attrs or (if these values are None)
+ # * parsed from the endpoint <service>.<region>.amazonaws.com.
+ region_name = self.determine_region_name(http_request.host)
+ service_name = self.determine_service_name(http_request.host)
http_request.service_name = service_name
http_request.region_name = region_name
@@ -516,6 +512,153 @@ def add_auth(self, req, **kwargs):
req.headers['Authorization'] = ','.join(l)
+class S3HmacAuthV4Handler(HmacAuthV4Handler, AuthHandler):
+ """
+ Implements a variant of Version 4 HMAC authorization specific to S3.
+ """
+ capability = ['hmac-v4-s3']
+
+ def __init__(self, *args, **kwargs):
+ super(S3HmacAuthV4Handler, self).__init__(*args, **kwargs)
+
+ if self.region_name:
+ self.region_name = self.clean_region_name(self.region_name)
+
+ def clean_region_name(self, region_name):
+ if region_name.startswith('s3-'):
+ return region_name[3:]
+
+ return region_name
+
+ def canonical_uri(self, http_request):
+ # S3 does **NOT** do path normalization that SigV4 typically does.
+ # Urlencode the path, **NOT** ``auth_path`` (because vhosting).
+ path = urlparse.urlparse(http_request.path)
+ encoded = urllib.quote(path.path)
+ return encoded
+
+ def host_header(self, host, http_request):
+ port = http_request.port
+ secure = http_request.protocol == 'https'
+ if ((port == 80 and not secure) or (port == 443 and secure)):
+ return http_request.host
+ return '%s:%s' % (http_request.host, port)
+
+ def headers_to_sign(self, http_request):
+ """
+ Select the headers from the request that need to be included
+ in the StringToSign.
+ """
+ host_header_value = self.host_header(self.host, http_request)
+ headers_to_sign = {}
+ headers_to_sign = {'Host': host_header_value}
+ for name, value in http_request.headers.items():
+ lname = name.lower()
+ # Hooray for the only difference! The main SigV4 signer only does
+ # ``Host`` + ``x-amz-*``. But S3 wants pretty much everything
+ # signed, except for authorization itself.
+ if not lname in ['authorization']:
+ headers_to_sign[name] = value
+ return headers_to_sign
+
+ def determine_region_name(self, host):
+ # S3's different format(s) of representing region/service from the
+ # rest of AWS makes this hurt too.
+ #
+ # Possible domain formats:
+ # - s3.amazonaws.com (Classic)
+ # - s3-us-west-2.amazonaws.com (Specific region)
+ # - bukkit.s3.amazonaws.com (Vhosted Classic)
+ # - bukkit.s3-ap-northeast-1.amazonaws.com (Vhosted specific region)
+ # - s3.cn-north-1.amazonaws.com.cn - (Bejing region)
+ # - bukkit.s3.cn-north-1.amazonaws.com.cn - (Vhosted Bejing region)
+ parts = self.split_host_parts(host)
+
+ if self.region_name is not None:
+ region_name = self.region_name
+ else:
+ # Classic URLs - s3-us-west-2.amazonaws.com
+ if len(parts) == 3:
+ region_name = self.clean_region_name(parts[0])
+
+ # Special-case for Classic.
+ if region_name == 's3':
+ region_name = 'us-east-1'
+ else:
+ # Iterate over the parts in reverse order.
+ for offset, part in enumerate(reversed(parts)):
+ part = part.lower()
+
+ # Look for the first thing starting with 's3'.
+ # Until there's a ``.s3`` TLD, we should be OK. :P
+ if part == 's3':
+ # If it's by itself, the region is the previous part.
+ region_name = parts[-offset]
+ break
+ elif part.startswith('s3-'):
+ region_name = self.clean_region_name(part)
+ break
+
+ return region_name
+
+ def determine_service_name(self, host):
+ # Should this signing mechanism ever be used for anything else, this
+ # will fail. Consider utilizing the logic from the parent class should
+ # you find yourself here.
+ return 's3'
+
+ def mangle_path_and_params(self, req):
+ """
+ Returns a copy of the request object with fixed ``auth_path/params``
+ attributes from the original.
+ """
+ modified_req = copy.copy(req)
+
+ # Unlike the most other services, in S3, ``req.params`` isn't the only
+ # source of query string parameters.
+ # Because of the ``query_args``, we may already have a query string
+ # **ON** the ``path/auth_path``.
+ # Rip them apart, so the ``auth_path/params`` can be signed
+ # appropriately.
+ parsed_path = urlparse.urlparse(modified_req.auth_path)
+ modified_req.auth_path = parsed_path.path
+
+ if modified_req.params is None:
+ modified_req.params = {}
+
+ raw_qs = parsed_path.query
+ existing_qs = urlparse.parse_qs(
+ raw_qs,
+ keep_blank_values=True
+ )
+
+ # ``parse_qs`` will return lists. Don't do that unless there's a real,
+ # live list provided.
+ for key, value in existing_qs.items():
+ if isinstance(value, (list, tuple)):
+ if len(value) == 1:
+ existing_qs[key] = value[0]
+
+ modified_req.params.update(existing_qs)
+ return modified_req
+
+ def payload(self, http_request):
+ if http_request.headers.get('x-amz-content-sha256'):
+ return http_request.headers['x-amz-content-sha256']
+
+ return super(S3HmacAuthV4Handler, self).payload(http_request)
+
+ def add_auth(self, req, **kwargs):
+ if not 'x-amz-content-sha256' in req.headers:
+ if '_sha256' in req.headers:
+ req.headers['x-amz-content-sha256'] = req.headers.pop('_sha256')
+ else:
+ req.headers['x-amz-content-sha256'] = self.payload(req)
+
+ req = self.mangle_path_and_params(req)
+ return super(S3HmacAuthV4Handler, self).add_auth(req, **kwargs)
+
+
class QueryAuthHandler(AuthHandler):
"""
Provides pure query construction (no actual signing).
@@ -742,3 +885,24 @@ def get_auth_handler(host, config, provider, requested_capability=None):
# user could override this with a .boto config that includes user-specific
# credentials (for access to user data).
return ready_handlers[-1]
+
+
+def detect_potential_sigv4(func):
+ def _wrapper(self):
+ if hasattr(self, 'region'):
+ if getattr(self.region, 'endpoint', ''):
+ if '.cn-' in self.region.endpoint:
+ return ['hmac-v4']
+
+ return func(self)
+ return _wrapper
+
+
+def detect_potential_s3sigv4(func):
+ def _wrapper(self):
+ if hasattr(self, 'host'):
+ if '.cn-' in self.host:
+ return ['hmac-v4-s3']
+
+ return func(self)
+ return _wrapper
Oops, something went wrong.

0 comments on commit 87a45cd

Please sign in to comment.