Skip to content
Sign up
This repository

public / boto Octocat-spinner-32

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Adding support for IAM Roles. Closes #811.

  • Loading branch information...
commit 304f2ed9cd1c0c3828cb511a7a94933e5b3c3088 1 parent bc05cb1
Mitch Garnaat authored

Showing 15 changed files with 627 additions and 36 deletions. Show Diff Stats Hide Diff Stats

  1. 46  boto/auth.py
  2. 40  boto/connection.py
  3. 20  boto/ec2/connection.py
  4. 3  boto/ec2/instance.py
  5. 121  boto/iam/connection.py
  6. 27  boto/provider.py
  7. 15  boto/route53/connection.py
  8. 13  boto/s3/connection.py
  9. 18  boto/ses/connection.py
  10. 10  boto/sns/connection.py
  11. 13  boto/sqs/connection.py
  12. 14  boto/swf/layer1.py
  13. 57  boto/utils.py
  14. 98  tests/ec2/test_instance.py
  15. 168  tests/misc/test_expire.py
46  boto/auth.py
View 
@@ -104,25 +104,29 @@ class AnonAuthHandler(AuthHandler, HmacKeys):
104 104 
     """
105 105 
     Implements Anonymous requests.
106 106 
     """
107   
-
  107 
+
108 108 
     capability = ['anon']
109   
-
  109 
+
110 110 
     def __init__(self, host, config, provider):
111 111 
         AuthHandler.__init__(self, host, config, provider)
112   
-
  112 
+
113 113 
     def add_auth(self, http_request, **kwargs):
114 114 
         pass
115 115
116 116 
 class HmacAuthV1Handler(AuthHandler, HmacKeys):
117 117 
     """    Implements the HMAC request signing used by S3 and GS."""
118   
-
  118 
+
119 119 
     capability = ['hmac-v1', 's3']
120   
-
  120 
+
121 121 
     def __init__(self, host, config, provider):
122 122 
         AuthHandler.__init__(self, host, config, provider)
123 123 
         HmacKeys.__init__(self, host, config, provider)
124 124 
         self._hmac_256 = None
125   
-
  125 
+
  126 
+    def update_provider(self, provider):
  127 
+        super(HmacAuthV1Handler, self).update_provider(provider)
  128 
+        self._hmac_256 = None
  129 
+
126 130 
     def add_auth(self, http_request, **kwargs):
127 131 
         headers = http_request.headers
128 132 
         method = http_request.method
 
@@ -148,12 +152,16 @@ class HmacAuthV2Handler(AuthHandler, HmacKeys):
148 152 
     Implements the simplified HMAC authorization used by CloudFront.
149 153 
     """
150 154 
     capability = ['hmac-v2', 'cloudfront']
151   
-
  155 
+
152 156 
     def __init__(self, host, config, provider):
153 157 
         AuthHandler.__init__(self, host, config, provider)
154 158 
         HmacKeys.__init__(self, host, config, provider)
155 159 
         self._hmac_256 = None
156   
-
  160 
+
  161 
+    def update_provider(self, provider):
  162 
+        super(HmacAuthV2Handler, self).update_provider(provider)
  163 
+        self._hmac_256 = None
  164 
+
157 165 
     def add_auth(self, http_request, **kwargs):
158 166 
         headers = http_request.headers
159 167 
         if 'Date' not in headers:
 
@@ -164,16 +172,16 @@ def add_auth(self, http_request, **kwargs):
164 172 
         headers['Authorization'] = ("%s %s:%s" %
165 173 
                                     (auth_hdr,
166 174 
                                      self._provider.access_key, b64_hmac))
167   
-
  175 
+
168 176 
 class HmacAuthV3Handler(AuthHandler, HmacKeys):
169 177 
     """Implements the new Version 3 HMAC authorization used by Route53."""
170   
-
  178 
+
171 179 
     capability = ['hmac-v3', 'route53', 'ses']
172   
-
  180 
+
173 181 
     def __init__(self, host, config, provider):
174 182 
         AuthHandler.__init__(self, host, config, provider)
175 183 
         HmacKeys.__init__(self, host, config, provider)
176   
-
  184 
+
177 185 
     def add_auth(self, http_request, **kwargs):
178 186 
         headers = http_request.headers
179 187 
         if 'Date' not in headers:
 
@@ -188,9 +196,9 @@ class HmacAuthV3HTTPHandler(AuthHandler, HmacKeys):
188 196 
     """
189 197 
     Implements the new Version 3 HMAC authorization used by DynamoDB.
190 198 
     """
191   
-
  199 
+
192 200 
     capability = ['hmac-v3-http']
193   
-
  201 
+
194 202 
     def __init__(self, host, config, provider):
195 203 
         AuthHandler.__init__(self, host, config, provider)
196 204 
         HmacKeys.__init__(self, host, config, provider)
 
@@ -234,7 +242,7 @@ def string_to_sign(self, http_request):
234 242 
                                     '',
235 243 
                                     http_request.body])
236 244 
         return string_to_sign, headers_to_sign
237   
-
  245 
+
238 246 
     def add_auth(self, req, **kwargs):
239 247 
         """
240 248 
         Add AWS3 authentication to a request.
 
@@ -367,7 +375,7 @@ def _calc_signature(self, params, verb, path, server_name):
367 375 
 class POSTPathQSV2AuthHandler(QuerySignatureV2AuthHandler, AuthHandler):
368 376 
     """
369 377 
     Query Signature V2 Authentication relocating signed query
370   
-    into the path and allowing POST requests with Content-Types.
  378 
+    into the path and allowing POST requests with Content-Types.
371 379 
     """
372 380
373 381 
     capability = ['mws']
 
@@ -401,7 +409,7 @@ def get_auth_handler(host, config, provider, requested_capability=None):
401 409 
     :type host: string
402 410 
     :param host: The name of the host
403 411
404   
-    :type config:
  412 
+    :type config:
405 413 
     :param config:
406 414
407 415 
     :type provider:
 
@@ -422,13 +430,13 @@ def get_auth_handler(host, config, provider, requested_capability=None):
422 430 
             ready_handlers.append(handler(host, config, provider))
423 431 
         except boto.auth_handler.NotReadyToAuthenticate:
424 432 
             pass
425   
-
  433 
+
426 434 
     if not ready_handlers:
427 435 
         checked_handlers = auth_handlers
428 436 
         names = [handler.__name__ for handler in checked_handlers]
429 437 
         raise boto.exception.NoAuthHandlerFound(
430 438 
               'No handler was ready to authenticate. %d handlers were checked.'
431   
-              ' %s '
  439 
+              ' %s '
432 440 
               'Check your credentials' % (len(names), str(names)))
433 441
434 442 
     if len(ready_handlers) > 1:
40  boto/connection.py
View 
@@ -55,6 +55,7 @@
55 55 
 import time
56 56 
 import urllib, urlparse
57 57 
 import xml.sax
  58 
+from xml.etree import ElementTree
58 59
59 60 
 import auth
60 61 
 import auth_handler
 
@@ -477,7 +478,8 @@ def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
477 478 
             # Allow overriding Provider
478 479 
             self.provider = provider
479 480 
         else:
480   
-            self.provider = Provider(provider,
  481 
+            self._provider_type = provider
  482 
+            self.provider = Provider(self._provider_type,
481 483 
                                      aws_access_key_id,
482 484 
                                      aws_secret_access_key,
483 485 
                                      security_token)
 
@@ -733,6 +735,10 @@ def _mexe(self, request, sender=None, override_num_retries=None,
733 735 
             num_retries = override_num_retries
734 736 
         i = 0
735 737 
         connection = self.get_http_connection(request.host, self.is_secure)
  738 
+        # The original headers/params are stored so that we can restore them
  739 
+        # if credentials are refreshed.
  740 
+        original_headers = request.headers.copy()
  741 
+        original_params = request.params.copy()
736 742 
         while i <= num_retries:
737 743 
             # Use binary exponential backoff to desynchronize client requests
738 744 
             next_sleep = random.random() * (2 ** i)
 
@@ -767,6 +773,13 @@ def _mexe(self, request, sender=None, override_num_retries=None,
767 773 
                     msg += 'Retrying in %3.1f seconds' % next_sleep
768 774 
                     boto.log.debug(msg)
769 775 
                     body = response.read()
  776 
+                elif self._credentials_expired(response):
  777 
+                    # The same request object is used so the security token and
  778 
+                    # access key params are cleared because they are no longer
  779 
+                    # valid.
  780 
+                    request.params = original_params.copy()
  781 
+                    request.headers = original_headers.copy()
  782 
+                    self._renew_credentials()
770 783 
                 elif response.status < 300 or response.status >= 400 or \
771 784 
                         not location:
772 785 
                     self.put_http_connection(request.host, self.is_secure,
 
@@ -809,6 +822,31 @@ def _mexe(self, request, sender=None, override_num_retries=None,
809 822 
             msg = 'Please report this exception as a Boto Issue!'
810 823 
             raise BotoClientError(msg)
811 824
  825 
+    def _credentials_expired(self, response):
  826 
+        # It is possible that we could be using temporary credentials that are
  827 
+        # now expired.  We want to detect when this happens so that we can
  828 
+        # refresh the credentials.  Subclasses can override this method and
  829 
+        # determine whether or not the response indicates that the credentials
  830 
+        # are invalid.  If this method returns True, the credentials will be
  831 
+        # renewed.
  832 
+        if response.status != 403:
  833 
+            return False
  834 
+        try:
  835 
+            for event, node in ElementTree.iterparse(response, events=['start']):
  836 
+                if node.tag.endswith('Code'):
  837 
+                    if node.text == 'ExpiredToken':
  838 
+                        return True
  839 
+        except ElementTree.ParseError:
  840 
+            return False
  841 
+        return False
  842 
+
  843 
+    def _renew_credentials(self):
  844 
+        # By resetting the provider with a new provider, this will trigger the
  845 
+        # lookup process for finding the new set of credentials.
  846 
+        boto.log.debug("Refreshing credentials.")
  847 
+        self.provider = Provider(self._provider_type)
  848 
+        self._auth_handler.update_provider(self.provider)
  849 
+
812 850 
     def build_base_http_request(self, method, path, auth_path,
813 851 
                                 params=None, headers=None, data='', host=None):
814 852 
         path = self.get_path(path)
20  boto/ec2/connection.py
View 
@@ -28,6 +28,8 @@
28 28 
 import warnings
29 29 
 from datetime import datetime
30 30 
 from datetime import timedelta
  31 
+from xml.etree import ElementTree
  32 
+
31 33 
 import boto
32 34 
 from boto.connection import AWSQueryConnection
33 35 
 from boto.resultset import ResultSet
 
@@ -60,7 +62,7 @@
60 62
61 63 
 class EC2Connection(AWSQueryConnection):
62 64
63   
-    APIVersion = boto.config.get('Boto', 'ec2_version', '2012-03-01')
  65 
+    APIVersion = boto.config.get('Boto', 'ec2_version', '2012-06-01')
64 66 
     DefaultRegionName = boto.config.get('Boto', 'ec2_region_name', 'us-east-1')
65 67 
     DefaultRegionEndpoint = boto.config.get('Boto', 'ec2_region_endpoint',
66 68 
                                             'ec2.us-east-1.amazonaws.com')
 
@@ -92,6 +94,15 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
92 94 
     def _required_auth_capability(self):
93 95 
         return ['ec2']
94 96
  97 
+    def _credentials_expired(self, response):
  98 
+        if response.status != 400:
  99 
+            return False
  100 
+        for event, node in ElementTree.iterparse(response, events=['start']):
  101 
+            if node.tag.endswith('Code'):
  102 
+                if node.text == 'RequestExpired':
  103 
+                    return True
  104 
+        return False
  105 
+
95 106 
     def get_params(self):
96 107 
         """
97 108 
         Returns a dictionary containing the value of of all of the keyword
 
@@ -520,7 +531,8 @@ def run_instances(self, image_id, min_count=1, max_count=1,
520 531 
                       private_ip_address=None,
521 532 
                       placement_group=None, client_token=None,
522 533 
                       security_group_ids=None,
523   
-                      additional_info=None, tenancy=None):
  534 
+                      additional_info=None, instance_profile_name=None,
  535 
+                      instance_profile_arn=None, tenancy=None):
524 536 
         """
525 537 
         Runs an image on EC2.
526 538 
 
@@ -688,6 +700,10 @@ def run_instances(self, image_id, min_count=1, max_count=1,
688 700 
             params['ClientToken'] = client_token
689 701 
         if additional_info:
690 702 
             params['AdditionalInfo'] = additional_info
  703 
+        if instance_profile_name:
  704 
+            params['IamInstanceProfile.Name'] = instance_profile_name
  705 
+        if instance_profile_arn:
  706 
+            params['IamInstanceProfile.Arn'] = instance_profile_arn
691 707 
         return self.get_object('RunInstances', params, Reservation, verb='POST')
692 708
693 709 
     def terminate_instances(self, instance_ids=None):
3  boto/ec2/instance.py
View 
@@ -184,6 +184,9 @@ def startElement(self, name, attrs, connection):
184 184 
             return self.eventsSet
185 185 
         elif name == 'networkInterfaceSet':
186 186 
             self.interfaces = ResultSet([('item', NetworkInterface)])
  187 
+        elif name == 'iamInstanceProfile':
  188 
+            self.instance_profile = SubParse('iamInstanceProfile')
  189 
+            return self.instance_profile
187 190 
         return None
188 191
189 192 
     def endElement(self, name, value, connection):
121  boto/iam/connection.py
View 
@@ -19,13 +19,20 @@
19 19 
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 20 
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 21 
 # IN THE SOFTWARE.
  22 
+import json
22 23
23 24 
 import boto
24 25 
 import boto.jsonresponse
  26 
+from boto.resultset import ResultSet
25 27 
 from boto.iam.summarymap import SummaryMap
26 28 
 from boto.connection import AWSQueryConnection
27 29
28   
-#boto.set_stream_logger('iam')
  30 
+
  31 
+ASSUME_ROLE_POLICY_DOCUMENT = json.dumps({
  32 
+    'Statement': [{'Principal': {'Service': ['ec2.amazonaws.com']},
  33 
+                   'Effect': 'Allow',
  34 
+                   'Action': ['sts:AssumeRole']}]})
  35 
+
29 36
30 37
31 38 
 class IAMConnection(AWSQueryConnection):
 
@@ -1006,3 +1013,115 @@ def get_account_summary(self):
1006 1013 
         http://goo.gl/ToB7G
1007 1014 
         """
1008 1015 
         return self.get_object('GetAccountSummary', {}, SummaryMap)
  1016 
+
  1017 
+    #
  1018 
+    # IAM Roles
  1019 
+    #
  1020 
+
  1021 
+    def add_role_to_instance_profile(self, instance_profile_name, role_name):
  1022 
+        return self.get_response('AddRoleToInstanceProfile',
  1023 
+                                 {'InstanceProfileName': instance_profile_name,
  1024 
+                                  'RoleName': role_name})
  1025 
+
  1026 
+    def create_instance_profile(self, instance_profile_name, path=None):
  1027 
+        params = {'InstanceProfileName': instance_profile_name}
  1028 
+        if path is not None:
  1029 
+            params['Path'] = path
  1030 
+        return self.get_response('CreateInstanceProfile', params)
  1031 
+
  1032 
+    def create_role(self, role_name, assume_role_policy_document=None, path=None):
  1033 
+        params = {'RoleName': role_name}
  1034 
+        if assume_role_policy_document is None:
  1035 
+            # This is the only valid assume_role_policy_document currently, so
  1036 
+            # this is used as a default value if no assume_role_policy_document
  1037 
+            # is provided.
  1038 
+            params['AssumeRolePolicyDocument'] =  ASSUME_ROLE_POLICY_DOCUMENT
  1039 
+        else:
  1040 
+            params['AssumeRolePolicyDocument'] =  assume_role_policy_document
  1041 
+        if path is not None:
  1042 
+            params['Path'] = path
  1043 
+        return self.get_response('CreateRole', params)
  1044 
+
  1045 
+    def delete_instance_profile(self, instance_profile_name):
  1046 
+        return self.get_response(
  1047 
+            'DeleteInstanceProfile',
  1048 
+            {'InstanceProfileName': instance_profile_name})
  1049 
+
  1050 
+    def delete_role(self, role_name):
  1051 
+        return self.get_response('DeleteRole', {'RoleName': role_name})
  1052 
+
  1053 
+    def delete_role_policy(self, role_name, policy_name):
  1054 
+        return self.get_response(
  1055 
+            'DeleteRolePolicy',
  1056 
+            {'RoleName': role_name, 'PolicyName': policy_name})
  1057 
+
  1058 
+    def get_instance_profile(self, instance_profile_name):
  1059 
+        return self.get_response('GetInstanceProfile', {'InstanceProfileName':
  1060 
+                                                       instance_profile_name})
  1061 
+
  1062 
+    def get_role(self, role_name):
  1063 
+        return self.get_response('GetRole', {'RoleName': role_name})
  1064 
+
  1065 
+    def get_role_policy(self, role_name, policy_name):
  1066 
+        return self.get_response('GetRolePolicy',
  1067 
+                                 {'RoleName': role_name,
  1068 
+                                  'PolicyName': policy_name})
  1069 
+
  1070 
+    def list_instance_profiles(self, path_prefix=None, marker=None,
  1071 
+                               max_items=None):
  1072 
+        params = {}
  1073 
+        if path_prefix is not None:
  1074 
+            params['PathPrefix'] = path_prefix
  1075 
+        if marker is not None:
  1076 
+            params['Marker'] = marker
  1077 
+        if max_items is not None:
  1078 
+            params['MaxItems'] = max_items
  1079 
+
  1080 
+        return self.get_response('ListInstanceProfiles', params,
  1081 
+                                 list_marker='InstanceProfiles')
  1082 
+
  1083 
+    def list_instance_profiles_for_role(self, role_name, marker=None,
  1084 
+                                        max_items=None):
  1085 
+        params = {'RoleName': role_name}
  1086 
+        if marker is not None:
  1087 
+            params['Marker'] = marker
  1088 
+        if max_items is not None:
  1089 
+            params['MaxItems'] = max_items
  1090 
+        return self.get_response('ListInstanceProfilesForRole', params,
  1091 
+                                 list_marker='InstanceProfiles')
  1092 
+
  1093 
+    def list_role_policies(self, role_name, marker=None, max_items=None):
  1094 
+        params = {'RoleName': role_name}
  1095 
+        if marker is not None:
  1096 
+            params['Marker'] = marker
  1097 
+        if max_items is not None:
  1098 
+            params['MaxItems'] = max_items
  1099 
+        return self.get_response('ListRolePolicies', params,
  1100 
+                                 list_marker='PolicyNames')
  1101 
+
  1102 
+    def list_roles(self, path_prefix=None, marker=None, max_items=None):
  1103 
+        params = {}
  1104 
+        if path_prefix is not None:
  1105 
+            params['PathPrefix'] = path_prefix
  1106 
+        if marker is not None:
  1107 
+            params['Marker'] = marker
  1108 
+        if max_items is not None:
  1109 
+            params['MaxItems'] = max_items
  1110 
+        return self.get_response('ListRoles', params, list_marker='Roles')
  1111 
+
  1112 
+    def put_role_policy(self, role_name, policy_name, policy_document):
  1113 
+        return self.get_response('PutRolePolicy',
  1114 
+                                 {'RoleName': role_name,
  1115 
+                                  'PolicyName': policy_name,
  1116 
+                                  'PolicyDocument': policy_document})
  1117 
+
  1118 
+    def remove_role_from_instance_profile(self, instance_profile_name,
  1119 
+                                          role_name):
  1120 
+        return self.get_response('RemoveRoleFromInstanceProfile',
  1121 
+                                 {'InstanceProfileName': instance_profile_name,
  1122 
+                                  'RoleName': role_name})
  1123 
+
  1124 
+    def update_assume_role_policy(self, role_name, policy_document):
  1125 
+        return self.get_response('UpdateAssumeRolePolicy',
  1126 
+                                 {'RoleName': role_name,
  1127 
+                                  'PolicyDocument': policy_document})
27  boto/provider.py
View 
@@ -89,6 +89,11 @@ class Provider(object):
89 89 
         'google': True
90 90 
     }
91 91
  92 
+    MetadataServiceSupport = {
  93 
+        'aws': True,
  94 
+        'google': False
  95 
+    }
  96 
+
92 97 
     # If you update this map please make sure to put "None" for the
93 98 
     # right-hand-side for any headers that don't apply to a provider, rather
94 99 
     # than simply leaving that header out (which would cause KeyErrors).
 
@@ -187,11 +192,33 @@ def get_credentials(self, access_key=None, secret_key=None):
187 192 
             self.secret_key = os.environ[secret_key_name.upper()]
188 193 
         elif config.has_option('Credentials', secret_key_name):
189 194 
             self.secret_key = config.get('Credentials', secret_key_name)
  195 
+
  196 
+        if ((self.access_key is None or self.secret_key is None) and
  197 
+                self.MetadataServiceSupport[self.name]):
  198 
+            self._populate_keys_from_metadata_server()
  199 
+
190 200 
         if isinstance(self.secret_key, unicode):
191 201 
             # the secret key must be bytes and not unicode to work
192 202 
             #  properly with hmac.new (see http://bugs.python.org/issue5285)
193 203 
             self.secret_key = str(self.secret_key)
194 204
  205 
+    def _populate_keys_from_metadata_server(self):
  206 
+        # get_instance_metadata is imported here because of a circular
  207 
+        # dependency.
  208 
+        from boto.utils import get_instance_metadata
  209 
+        timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
  210 
+        credentials = get_instance_metadata(timeout=timeout)
  211 
+        if credentials is None:
  212 
+            return
  213 
+        # I'm assuming there's only one role on the instance profile.
  214 
+        security = credentials['iam']['security-credentials'].values()[0]
  215 
+        if self.access_key is None:
  216 
+            self.access_key = security['AccessKeyId']
  217 
+        if self.secret_key is None:
  218 
+            self.secret_key = security['SecretAccessKey']
  219 
+        if self.security_token is None:
  220 
+            self.security_token = security['Token']
  221 
+
195 222 
     def configure_headers(self):
196 223 
         header_info_map = self.HeaderInfoMap[self.name]
197 224 
         self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY]
15  boto/route53/connection.py
View 
@@ -20,11 +20,12 @@
20 20 
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 21 
 # IN THE SOFTWARE.
22 22 
 #
23   
-
24 23 
 import xml.sax
25 24 
 import time
26 25 
 import uuid
27 26 
 import urllib
  27 
+from xml.etree import ElementTree
  28 
+
28 29 
 import boto
29 30 
 from boto.connection import AWSAuthConnection
30 31 
 from boto import handler
 
@@ -66,6 +67,18 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
66 67 
     def _required_auth_capability(self):
67 68 
         return ['route53']
68 69
  70 
+    def _credentials_expired(self, response):
  71 
+        if response.status != 403:
  72 
+            return False
  73 
+        try:
  74 
+            for event, node in ElementTree.iterparse(response, events=['start']):
  75 
+                if node.tag.endswith('Code'):
  76 
+                    if node.text == 'InvalidClientTokenId':
  77 
+                        return True
  78 
+        except ElementTree.ParseError:
  79 
+            return False
  80 
+        return False
  81 
+
69 82 
     def make_request(self, action, path, headers=None, data='', params=None):
70 83 
         if params:
71 84 
             pairs = []
13  boto/s3/connection.py
View 
@@ -22,6 +22,7 @@
22 22 
 # IN THE SOFTWARE.
23 23
24 24 
 import xml.sax
  25 
+from xml.etree import ElementTree
25 26 
 import urllib
26 27 
 import base64
27 28 
 import time
 
@@ -171,6 +172,18 @@ def _required_auth_capability(self):
171 172 
         else:
172 173 
             return ['s3']
173 174
  175 
+    def _credentials_expired(self, response):
  176 
+        if response.status != 400:
  177 
+            return False
  178 
+        try:
  179 
+            for event, node in ElementTree.iterparse(response, events=['start']):
  180 
+                if node.tag.endswith('Code'):
  181 
+                    if node.text == 'ExpiredToken':
  182 
+                        return True
  183 
+        except ElementTree.ParseError:
  184 
+            return False
  185 
+        return False
  186 
+
174 187 
     def __iter__(self):
175 188 
         for bucket in self.get_all_buckets():
176 189 
             yield bucket
18  boto/ses/connection.py
View 
@@ -19,15 +19,15 @@
19 19 
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 20 
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 21 
 # IN THE SOFTWARE.
  22 
+import urllib
  23 
+import base64
  24 
+from xml.etree import ElementTree
22 25
23 26 
 from boto.connection import AWSAuthConnection
24 27 
 from boto.exception import BotoServerError
25 28 
 from boto.regioninfo import RegionInfo
26 29 
 import boto
27 30 
 import boto.jsonresponse
28   
-
29   
-import urllib
30   
-import base64
31 31 
 from boto.ses import exceptions as ses_exceptions
32 32
33 33 
 
@@ -57,6 +57,18 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
57 57 
     def _required_auth_capability(self):
58 58 
         return ['ses']
59 59
  60 
+    def _credentials_expired(self, response):
  61 
+        if response.status != 403:
  62 
+            return False
  63 
+        try:
  64 
+            for event, node in ElementTree.iterparse(response, events=['start']):
  65 
+                if node.tag.endswith('Code'):
  66 
+                    if node.text == 'InvalidClientTokenId':
  67 
+                        return True
  68 
+        except ElementTree.ParseError:
  69 
+            return False
  70 
+        return False
  71 
+
60 72 
     def _build_list_params(self, params, items, label):
61 73 
         """Add an AWS API-compatible parameter list to a dictionary.
62 74
10  boto/sns/connection.py
View 
@@ -55,6 +55,16 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
55 55 
     def _required_auth_capability(self):
56 56 
         return ['sns']
57 57
  58 
+    def _credentials_expired(self, response):
  59 
+        if response.status != 403:
  60 
+            return False
  61 
+        try:
  62 
+            parsed = json.loads(response.read())
  63 
+            return parsed['Error']['Code'] == 'ExpiredToken'
  64 
+        except Exception:
  65 
+            return False
  66 
+        return False
  67 
+
58 68 
     def get_all_topics(self, next_token=None):
59 69 
         """
60 70 
         :type next_token: string
13  boto/sqs/connection.py
View 
@@ -18,6 +18,7 @@
18 18 
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 19 
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20 20 
 # IN THE SOFTWARE.
  21 
+from xml.etree import ElementTree
21 22
22 23 
 from boto.connection import AWSQueryConnection
23 24 
 from boto.sqs.regioninfo import SQSRegionInfo
 
@@ -59,6 +60,18 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
59 60 
     def _required_auth_capability(self):
60 61 
         return ['sqs']
61 62
  63 
+    def _credentials_expired(self, response):
  64 
+        if response.status != 401:
  65 
+            return False
  66 
+        try:
  67 
+            for event, node in ElementTree.iterparse(response, events=['start']):
  68 
+                if node.tag.endswith('Code'):
  69 
+                    if node.text == 'InvalidAccessKeyId':
  70 
+                        return True
  71 
+        except ElementTree.ParseError:
  72 
+            return False
  73 
+        return False
  74 
+
62 75 
     def create_queue(self, queue_name, visibility_timeout=None):
63 76 
         """
64 77 
         Create an SQS Queue.
14  boto/swf/layer1.py
View 
@@ -87,6 +87,20 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
87 87 
     def _required_auth_capability(self):
88 88 
         return ['hmac-v3-http']
89 89
  90 
+    def _credentials_expired(self, response):
  91 
+        if response.status != 400:
  92 
+            return False
  93 
+        try:
  94 
+            parsed = json.loads(response.read())
  95 
+            # It seems that SWF doesn't really have a specific "Token Expired"
  96 
+            # message, but this is a best effort heuristic.
  97 
+            expected = 'com.amazon.coral.service#UnrecognizedClientException'
  98 
+            return (parsed['__type'] == expected and \
  99 
+                    'security token' in parsed['message'])
  100 
+        except Exception, e:
  101 
+            return False
  102 
+        return False
  103 
+
90 104 
     def make_request(self, action, body='', object_hook=None):
91 105 
         """
92 106 
         :raises: ``SWFResponseError`` if response status is not 200.
57  boto/utils.py
View 
@@ -38,6 +38,7 @@
38 38 
 Some handy utility functions used by several classes.
39 39 
 """
40 40
  41 
+import socket
41 42 
 import urllib
42 43 
 import urllib2
43 44 
 import imp
 
@@ -70,6 +71,11 @@
70 71 
     import md5
71 72 
     _hashfn = md5.md5
72 73
  74 
+try:
  75 
+    import simplejson as json
  76 
+except:
  77 
+    import json
  78 
+
73 79 
 # List of Query String Arguments of Interest
74 80 
 qsa_of_interest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
75 81 
                    'partNumber', 'policy', 'requestPayment', 'torrent', 
@@ -183,20 +189,36 @@ def retry_url(url, retry_on_404=True, num_retries=10):
183 189 
                 code = e.code
184 190 
             if code == 404 and not retry_on_404:
185 191 
                 return ''
186   
-        except:
  192 
+        except urllib2.URLError, e:
  193 
+            raise e
  194 
+        except Exception, e:
187 195 
             pass
188 196 
         boto.log.exception('Caught exception reading instance data')
189 197 
         time.sleep(2**i)
190 198 
     boto.log.error('Unable to read instance data, giving up')
191 199 
     return ''
192 200
  201 
+def _get_iam_instance_metadata(url):
  202 
+    d = {}
  203 
+    # get info first
  204 
+    data = retry_url(url + 'info', num_retries=1)
  205 
+    d['info'] = json.loads(data)
  206 
+    d['security-credentials'] = {}
  207 
+    cred_name = retry_url(url + 'security-credentials', num_retries=1)
  208 
+    data = retry_url(url + 'security-credentials' + '/' + cred_name,
  209 
+                     num_retries=1)
  210 
+    d['security-credentials'][cred_name] = json.loads(data)
  211 
+    return d
  212 
+
193 213 
 def _get_instance_metadata(url):
194 214 
     d = {}
195   
-    data = retry_url(url)
  215 
+    data = retry_url(url, num_retries=1)
196 216 
     if data:
197 217 
         fields = data.split('\n')
198 218 
         for field in fields:
199   
-            if field.endswith('/'):
  219 
+            if field == 'iam/':
  220 
+                d[field[0:-1]] = _get_iam_instance_metadata(url + field)
  221 
+            elif field.endswith('/'):
200 222 
                 d[field[0:-1]] = _get_instance_metadata(url + field)
201 223 
             else:
202 224 
                 p = field.find('=')
 
@@ -205,22 +227,39 @@ def _get_instance_metadata(url):
205 227 
                     resource = field[0:p] + '/openssh-key'
206 228 
                 else:
207 229 
                     key = resource = field
208   
-                val = retry_url(url + resource)
209   
-                p = val.find('\n')
210   
-                if p > 0:
211   
-                    val = val.split('\n')
  230 
+                val = retry_url(url + resource, num_retries=1)
  231 
+                if val[0] == '{':
  232 
+                    val = json.loads(val)
  233 
+                else:
  234 
+                    p = val.find('\n')
  235 
+                    if p > 0:
  236 
+                        val = val.split('\n')
212 237 
                 d[key] = val
213 238 
     return d
214 239
215   
-def get_instance_metadata(version='latest', url='http://169.254.169.254'):
  240 
+def get_instance_metadata(version='latest', url='http://169.254.169.254',
  241 
+                          timeout=None):
216 242 
     """
217 243 
     Returns the instance metadata as a nested Python dictionary.
218 244 
     Simple values (e.g. local_hostname, hostname, etc.) will be
219 245 
     stored as string values.  Values such as ancestor-ami-ids will
220 246 
     be stored in the dict as a list of string values.  More complex
221 247 
     fields such as public-keys and will be stored as nested dicts.
  248 
+
  249 
+    If the timeout is specified, the connection to the specified url
  250 
+    will time out after the specified number of seconds.
  251 
+
222 252 
     """
223   
-    return _get_instance_metadata('%s/%s/meta-data/' % (url, version))
  253 
+    if timeout is not None:
  254 
+        original = socket.getdefaulttimeout()
  255 
+        socket.setdefaulttimeout(timeout)
  256 
+    try:
  257 
+        return _get_instance_metadata('%s/%s/meta-data/' % (url, version))
  258 
+    except urllib2.URLError, e:
  259 
+        return None
  260 
+    finally:
  261 
+        if timeout is not None:
  262 
+            socket.setdefaulttimeout(original)
224 263
225 264 
 def get_instance_userdata(version='latest', sep=None,
226 265 
                           url='http://169.254.169.254'):
98  tests/ec2/test_instance.py
View
... ... 
@@ -0,0 +1,98 @@
  1 
+#!/usr/bin/env python
  2 
+
  3 
+import unittest
  4 
+
  5 
+import mock
  6 
+
  7 
+from boto.ec2.connection import EC2Connection
  8 
+
  9 
+
  10 
+RESPONSE = r"""
  11 
+<RunInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2012-06-01/">
  12 
+    <requestId>ad4b83c2-f606-4c39-90c6-5dcc5be823e1</requestId>
  13 
+    <reservationId>r-c5cef7a7</reservationId>
  14 
+    <ownerId>184906166255</ownerId>
  15 
+    <groupSet>
  16 
+        <item>
  17 
+            <groupId>sg-99a710f1</groupId>
  18 
+            <groupName>SSH</groupName>
  19 
+        </item>
  20 
+    </groupSet>
  21 
+    <instancesSet>
  22 
+        <item>
  23 
+            <instanceId>i-ff0f1299</instanceId>
  24 
+            <imageId>ami-ed65ba84</imageId>
  25 
+            <instanceState>
  26 
+                <code>0</code>
  27 
+                <name>pending</name>
  28 
+            </instanceState>
  29 
+            <privateDnsName/>
  30 
+            <dnsName/>
  31 
+            <reason/>
  32 
+            <keyName>awskeypair</keyName>
  33 
+            <amiLaunchIndex>0</amiLaunchIndex>
  34 
+            <productCodes/>
  35 
+            <instanceType>t1.micro</instanceType>
  36 
+            <launchTime>2012-05-30T19:21:18.000Z</launchTime>
  37 
+            <placement>
  38 
+                <availabilityZone>us-east-1a</availabilityZone>
  39 
+                <groupName/>
  40 
+                <tenancy>default</tenancy>
  41 
+            </placement>
  42 
+            <kernelId>aki-b6aa75df</kernelId>
  43 
+            <monitoring>
  44 
+                <state>disabled</state>
  45 
+            </monitoring>
  46 
+            <groupSet>
  47 
+                <item>
  48 
+                    <groupId>sg-99a710f1</groupId>
  49 
+                    <groupName>SSH</groupName>
  50 
+                </item>
  51 
+            </groupSet>
  52 
+            <stateReason>
  53 
+                <code>pending</code>
  54 
+                <message>pending</message>
  55 
+            </stateReason>
  56 
+            <architecture>i386</architecture>
  57 
+            <rootDeviceType>ebs</rootDeviceType>
  58 
+            <rootDeviceName>/dev/sda1</rootDeviceName>
  59 
+            <blockDeviceMapping/>
  60 
+            <virtualizationType>paravirtual</virtualizationType>
  61 
+            <clientToken/>
  62 
+            <hypervisor>xen</hypervisor>
  63 
+            <networkInterfaceSet/>
  64 
+            <iamInstanceProfile>
  65 
+                <arn>arn:aws:iam::184906166255:instance-profile/myinstanceprofile</arn>
  66 
+                <id>AIPAIQ2LVHYBCH7LYQFDK</id>
  67 
+            </iamInstanceProfile>
  68 
+        </item>
  69 
+    </instancesSet>
  70 
+</RunInstancesResponse>
  71 
+"""
  72 
+
  73 
+
  74 
+class TestRunInstanceResponseParsing(unittest.TestCase):
  75 
+    def testIAMInstanceProfileParsedCorrectly(self):
  76 
+        ec2 = EC2Connection(aws_access_key_id='aws_access_key_id',
  77 
+                            aws_secret_access_key='aws_secret_access_key')
  78 
+        mock_response = mock.Mock()
  79 
+        mock_response.read.return_value = RESPONSE
  80 
+        mock_response.status = 200
  81 
+        ec2.make_request = mock.Mock(return_value=mock_response)
  82 
+        reservation = ec2.run_instances(image_id='ami-12345')
  83 
+        self.assertEqual(len(reservation.instances), 1)
  84 
+        instance = reservation.instances[0]
  85 
+        self.assertEqual(instance.image_id, 'ami-ed65ba84')
  86 
+        # iamInstanceProfile has an ID element, so we want to make sure
  87 
+        # that this does not map to instance.id (which should be the
  88 
+        # id of the ec2 instance).
  89 
+        self.assertEqual(instance.id, 'i-ff0f1299')
  90 
+        self.assertDictEqual(
  91 
+            instance.instance_profile,
  92 
+            {'arn': ('arn:aws:iam::184906166255:'
  93 
+                     'instance-profile/myinstanceprofile'),
  94 
+             'id': 'AIPAIQ2LVHYBCH7LYQFDK'})
  95 
+
  96 
+
  97 
+if __name__ == '__main__':
  98 
+    unittest.main()
168  tests/misc/test_expire.py
View
... ... 
@@ -0,0 +1,168 @@
  1 
+#!/usr/bin/env python
  2 
+
  3 
+import unittest
  4 
+from StringIO import StringIO
  5 
+
  6 
+from boto.ec2.connection import EC2Connection
  7 
+from boto.ec2.elb import ELBConnection
  8 
+from boto.route53.connection import Route53Connection
  9 
+from boto.ses.connection import SESConnection
  10 
+from boto.sqs.connection import SQSConnection
  11 
+from boto.s3.connection import S3Connection
  12 
+from boto.sns.connection import SNSConnection
  13 
+from boto.cloudsearch.layer1 import Layer1 as CloudSearchConnection
  14 
+from boto.swf.layer1 import Layer1 as SWFConnection
  15 
+
  16 
+
  17 
+GENERIC_BAD_REQUEST = (
  18 
+    '<?xml version="1.0" encoding="UTF-8"?>\n'
  19 
+    '<Response><Errors><Error><Code>Invalid Request</Code>'
  20 
+    '<Message>Unknown Parameter.</Message></Error></Errors>'
  21 
+    '<RequestID>ab8cd8bd-bf38-451e-a61f-b007a336427b</RequestID></Response>'
  22 
+)
  23 
+
  24 
+EC2_EXPIRED = (
  25 
+    '<?xml version="1.0" encoding="UTF-8"?>\n'
  26 
+    '<Response><Errors><Error><Code>RequestExpired</Code>'
  27 
+    '<Message>Request has expired.</Message></Error></Errors>'
  28 
+    '<RequestID>ab8cd8bd-bf38-451e-a61f-b007a336427a</RequestID></Response>'
  29 
+)
  30 
+
  31 
+ROUTE_53_EXPIRED = (
  32 
+    '<?xml version="1.0"?>\n'
  33 
+    '<ErrorResponse xmlns="https://route53.amazonaws.com/doc/2012-02-29/">'
  34 
+    '<Error><Type>Sender</Type><Code>InvalidClientTokenId</Code>'
  35 
+    '<Message>The security token included in the request is invalid</Message>'
  36 
+    '</Error><RequestId>'
  37 
+    '6ae75e39-ae50-11e1-bb74-6f99ae4b5ac7</RequestId></ErrorResponse>'
  38 
+)
  39 
+
  40 
+SES_EXPIRED = (
  41 
+    '<ErrorResponse xmlns="http://ses.amazonaws.com/doc/2010-12-01/">\n'
  42 
+    '  <Error>\n'
  43 
+    '    <Type>Sender</Type>\n'
  44 
+    '    <Code>InvalidClientTokenId</Code>\n'
  45 
+    '    <Message>The security token included in the request'
  46 
+    'is invalid</Message>\n'
  47 
+    '  </Error>\n'
  48 
+    '  <RequestId>19e6091f-ae53-11e1-83f2-e9ee6fcf8644</RequestId>\n'
  49 
+    '</ErrorResponse>'
  50 
+)
  51 
+
  52 
+SQS_EXPIRED = (
  53 
+    '<?xml version="1.0"?>\n'
  54 
+    '<ErrorResponse xmlns="http://queue.amazonaws.com/doc/2011-10-01/">'
  55 
+    '<Error><Type>Sender</Type><Code>InvalidAccessKeyId</Code>'
  56 
+    '<Message>AWS was not able to validate the provided'
  57 
+    'access credentials.</Message><Detail/></Error>'
  58 
+    '<RequestId>9e59388c-d590-4750-bf4b-7f5c6f22069e</RequestId>'
  59 
+    '</ErrorResponse>'
  60 
+)
  61 
+
  62 
+S3_EXPIRED = (
  63 
+    '<?xml version="1.0" encoding="UTF-8"?>\n'
  64 
+    '<Error><Code>ExpiredToken</Code>'
  65 
+    '<Message>The provided token has expired.</Message>'
  66 
+    '<RequestId>8F73002DD7CE6806</RequestId>'
  67 
+    '<HostId>Y3X4aRVBW</HostId>'
  68 
+    '<Token-0>+3jZbqJRcmn4FTAslS/vCDq29dfpOGlcGiesXvSDh4bL+BA=='
  69 
+    '</Token-0></Error>'
  70 
+)
  71 
+
  72 
+SNS_EXPIRED = (
  73 
+    '{"Error":{"Code":"ExpiredToken",'
  74 
+    '"Message":"The security token included in the '
  75 
+    'request is expired","Type":"Sender"},'
  76 
+    '"RequestId":"49da0955-ae53-11e1-a190-fd9a9336a444"}'
  77 
+)
  78 
+
  79 
+SWF_EXPIRED = (
  80 
+    '{"message": "The security token included in the request is invalid",'
  81 
+    '"__type": "com.amazon.coral.service#UnrecognizedClientException"}'
  82 
+)
  83 
+
  84 
+# The majority of services (with the exception of the xmlns attr) have
  85 
+# an error response like this: autoscale, cloudformation, cloudsearch
  86 
+#
  87 
+GENERAL_EXPIRED = (
  88 
+    '<ErrorResponse xmlns="http://elasticloadbalancing.amazonaws.com/'
  89 
+    'doc/2011-11-15/">\n'
  90 
+    '  <Error>\n'
  91 
+    '    <Type>Sender</Type>\n'
  92 
+    '    <Code>ExpiredToken</Code>\n'
  93 
+    '    <Message>The security token included in the request'
  94 
+    'is expired</Message>\n'
  95 
+    '  </Error>\n'
  96 
+    '  <RequestId>7dfbf919-ae4f-11e1-bc8c-039e0b6d4242</RequestId>\n'
  97 
+    '</ErrorResponse>\n'
  98 
+)
  99 
+
  100 
+
  101 
+class FakeResponse(StringIO):
  102 
+    def __init__(self, content, status):
  103 
+        StringIO.__init__(self, content)
  104 
+        self.status = status
  105 
+
  106 
+
  107 
+class TestExpire(unittest.TestCase):
  108 
+    def assert_is_expired(self, connection, response, status):
  109 
+        self.assertTrue(connection._credentials_expired(
  110 
+            FakeResponse(response, status=status)),
  111 
+            "Connection was suppose to be expired but was not: %s" % response)
  112 
+
  113 
+    def assert_is_not_expired(self, connection, response, status):
  114 
+        self.assertFalse(connection._credentials_expired(
  115 
+            FakeResponse(response, status=status)),
  116 
+            "Connection was NOT suppose to be expired but was: %s" % response)
  117 
+
  118 
+    def test_ec2_expiration(self):
  119 
+        c = EC2Connection()
  120 
+        self.assert_is_expired(c, EC2_EXPIRED, status=400)
  121 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=400)
  122 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=403)
  123 
+
  124 
+    def test_elb_expiration(self):
  125 
+        c = ELBConnection()
  126 
+        self.assert_is_expired(c, GENERAL_EXPIRED, status=403)
  127 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=403)
  128 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=400)
  129 
+
  130 
+    def test_cloudsearch_expiration(self):
  131 
+        c = CloudSearchConnection()
  132 
+        self.assert_is_expired(c, GENERAL_EXPIRED, status=403)
  133 
+
  134 
+    def test_route_53_expiration(self):
  135 
+        c = Route53Connection()
  136 
+        self.assert_is_expired(c, ROUTE_53_EXPIRED, status=403)
  137 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=403)
  138 
+
  139 
+    def test_ses_expiration(self):
  140 
+        c = SESConnection()
  141 
+        self.assert_is_expired(c, SES_EXPIRED, status=403)
  142 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=403)
  143 
+
  144 
+    def test_sqs_expiration(self):
  145 
+        c = SQSConnection()
  146 
+        self.assert_is_expired(c, SQS_EXPIRED, status=401)
  147 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=403)
  148 
+
  149 
+    def test_s3_expired(self):
  150 
+        c = S3Connection()
  151 
+        self.assert_is_expired(c, S3_EXPIRED, status=400)
  152 
+        self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=400)
  153 
+
  154 
+    def test_sns_expired(self):
  155 
+        c = SNSConnection()
  156 
+        self.assert_is_expired(c, SNS_EXPIRED, status=403)
  157 
+
  158 
+    def test_swf_expired(self):
  159 
+        c = SWFConnection()
  160 
+        self.assert_is_expired(c, SWF_EXPIRED, status=400)
  161 
+
  162 
+    def test_non_xml_response(self):
  163 
+        c = CloudSearchConnection()
  164 
+        self.assert_is_not_expired(c, "{'this is': 'json'}", status=403)
  165 
+
  166 
+
  167 
+if __name__ == '__main__':
  168 
+    unittest.main()

0 notes on commit 304f2ed

Please sign in to comment.
Something went wrong with that request. Please try again.