Migrate the core of bespin to boto3

bespin/amazon/cloudformation.py

@@ -2,9 +2,11 @@
 from bespin.amazon.mixin import AmazonMixin
 from bespin import helpers as hp
 
-import boto.cloudformation
+import botocore
+import boto3
 import datetime
 import logging
+import pytz
 import time
 import six
 import os
@@ -58,11 +60,15 @@ class UPDATE_ROLLBACK_FAILED(Status): pass
 class UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS(Status): pass
 class UPDATE_ROLLBACK_COMPLETE(Status): pass
 
+# REVIEW_IN_PROGRESS only valid for CreateChangeSet with ChangeSetType=CREATE
+class REVIEW_IN_PROGRESS(Status): pass
+
 for kls in [Status] + Status.__subclasses__():
     with_meta = six.add_metaclass(StatusMeta)(kls)
     locals()[kls.__name__] = with_meta
     Status.statuses[kls.__name__] = with_meta
 
+##BOTO3 TODO: refactor to use boto3 resources
 class Cloudformation(AmazonMixin):
     def __init__(self, stack_name, region="ap-southeast-2"):
         self.region = region
@@ -71,7 +77,11 @@ def __init__(self, stack_name, region="ap-southeast-2"):
     @hp.memoized_property
     def conn(self):
         log.info("Using region [%s] for cloudformation (%s)", self.region, self.stack_name)
-        return boto.cloudformation.connect_to_region(self.region)
+        return self.session.client('cloudformation', region_name=self.region)
+
+    @hp.memoized_property
+    def session(self):
+        return boto3.session.Session(region_name=self.region)
 
     def reset(self):
         self._description = None
@@ -83,7 +93,8 @@ def description(self, force=False):
                 while True:
                     try:
                         with self.ignore_throttling_error():
-                            self._description = self.conn.describe_stacks(self.stack_name)[0]
+                            response = self.conn.describe_stacks(StackName=self.stack_name)
+                            self._description = response['Stacks'][0]
                             break
                     except Throttled:
                         log.info("Was throttled, waiting a bit")
@@ -94,10 +105,10 @@ def description(self, force=False):
     def outputs(self):
         self.wait()
         description = self.description()
-        if description is None:
-            return {}
+        if 'Outputs' in description.outputs:
+            return dict((out['OutputKey'], out['OutputValue']) for out in description['Outputs'])
         else:
-            return dict((out.key, out.value) for out in description.outputs)
+            return {}
 
     @property
     def status(self):
@@ -113,30 +124,32 @@ def status(self):
 
         try:
             description = self.description(force=force)
-            return Status.find(description.stack_status)
+            return Status.find(description['StackStatus'])
         except StackDoesntExist:
             return NONEXISTANT
 
     def map_logical_to_physical_resource_id(self, logical_id):
-        resource = self.conn.describe_stack_resource(stack_name_or_id=self.stack_name, logical_resource_id=logical_id)
-        return resource['DescribeStackResourceResponse']['DescribeStackResourceResult']['StackResourceDetail']["PhysicalResourceId"]
+        response = self.conn.describe_stack_resource(StackName=self.stack_name, LogicalResourceId=logical_id)
+        return response['StackResourceDetail']["PhysicalResourceId"]
+
+    def _convert_tags(self, tags):
+        """ helper to convert python dictionary into list of AWS Tag dicts """
+        return [{'Key': k, 'Value': v} for k,v in tags.items()] if tags else None
 
     def create(self, stack, params, tags):
         log.info("Creating stack (%s)\ttags=%s", self.stack_name, tags)
-        params = [(param["ParameterKey"], param["ParameterValue"]) for param in params] if params else None
         disable_rollback = os.environ.get("DISABLE_ROLLBACK", 0) == "1"
-        self.conn.create_stack(self.stack_name, template_body=stack, parameters=params, tags=tags, capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], disable_rollback=disable_rollback)
+        self.conn.create_stack(StackName=self.stack_name, TemplateBody=stack, Parameters=params, Tags=self._convert_tags(tags), Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], DisableRollback=disable_rollback)
         return True
 
-    def update(self, stack, params):
+    def update(self, stack, params, tags):
         log.info("Updating stack (%s)", self.stack_name)
-        params = [(param["ParameterKey"], param["ParameterValue"]) for param in params] if params else None
-        disable_rollback = os.environ.get("DISABLE_ROLLBACK", 0) == "1"
+        # NOTE: DisableRollback is not supported by UpdateStack. It is a property of the stack that can only be set during stack creation
         with self.catch_boto_400(BadStack, "Couldn't update the stack", stack_name=self.stack_name):
             try:
-                self.conn.update_stack(self.stack_name, template_body=stack, parameters=params, capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], disable_rollback=disable_rollback)
-            except boto.exception.BotoServerError as error:
-                if error.message == "No updates are to be performed.":
+                self.conn.update_stack(StackName=self.stack_name, TemplateBody=stack, Parameters=params, Tags=self._convert_tags(tags), Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'])
+            except botocore.exceptions.ClientError as error:
+                if error.response['Error']['Message'] == "No updates are to be performed.":
                     log.info("No updates were necessary!")
                     return False
                 else:
@@ -145,14 +158,16 @@ def update(self, stack, params):
 
     def validate_template(self, filename):
         with self.catch_boto_400(BadStack, "Amazon says no", stack_name=self.stack_name, filename=filename):
-            self.conn.validate_template(open(filename).read())
+            self.conn.validate_template(TemplateBody=open(filename).read())
 
+    ##BOTO3 TODO: can this be refactored with client.get_waiter?
+    ##BOTO3 TODO: also consider client.get_paginator('describe_stack_events')
     def wait(self, timeout=1200, rollback_is_failure=False, may_not_exist=True):
         status = self.status
         if not status.exists and may_not_exist:
             return status
 
-        last = datetime.datetime.utcnow()
+        last = datetime.datetime.now(pytz.utc)
         if status.failed:
             raise BadStack("Stack is in a failed state, it must be deleted first", name=self.stack_name, status=status)
 
@@ -172,17 +187,18 @@ def wait(self, timeout=1200, rollback_is_failure=False, may_not_exist=True):
             while True:
                 try:
                     with self.ignore_throttling_error():
-                        events = description.describe_events()
+                        response = self.conn.describe_stack_events(StackName=self.stack_name)
+                        events = response['StackEvents']
                         break
                 except Throttled:
                     log.info("Was throttled, waiting a bit")
                     time.sleep(1)
 
-            next_last = events[0].timestamp
+            next_last = events[0]['Timestamp']
             for event in events:
-                if event.timestamp > last:
-                    reason = event.resource_status_reason or ""
-                    log.info("%s - %s %s (%s) %s", self.stack_name, event.resource_type, event.logical_resource_id, event.resource_status, reason)
+                if event['Timestamp'] > last:
+                    reason = event['ResourceStatusReason'] or ""
+                    log.info("%s - %s %s (%s) %s", self.stack_name, event['ResourceType'], event['LogicalResourceId'], event['ResourceStatus'], reason)
             last = next_last
 
         status = self.status

bespin/amazon/credentials.py

@@ -1,19 +1,14 @@
 from bespin.amazon.cloudformation import Cloudformation
 from bespin.helpers import memoized_property
-from bespin.errors import BespinError
+from bespin.errors import BespinError, ProgrammerError
 from bespin.amazon.ec2 import EC2
 from bespin.amazon.sqs import SQS
 from bespin.amazon.kms import KMS
 from bespin.amazon.s3 import S3
-
-import boto.sts
-import boto.iam
-import boto.s3
-import boto.sqs
+from bespin import VERSION
 
 from input_algorithms.spec_base import NotSpecified
 import logging
-import boto
 import botocore
 import boto3
 import os
@@ -25,6 +20,7 @@ def __init__(self, region, account_id, assume_role):
         self.region = region
         self.account_id = account_id
         self.assume_role = assume_role
+        self.session = None
         self.clouds = {}
 
     def verify_creds(self):
@@ -39,7 +35,8 @@ def verify_creds(self):
 
         log.info("Verifying amazon credentials")
         try:
-            amazon_account_id = boto3.client('sts').get_caller_identity().get('Account')
+            self.session = boto3.session.Session(region_name=self.region)
+            amazon_account_id = self.session.client('sts').get_caller_identity().get('Account')
             if int(self.account_id) != int(amazon_account_id):
                 raise BespinError("Please use credentials for the right account", expect=self.account_id, got=amazon_account_id)
             self._verified = True
@@ -48,6 +45,9 @@ def verify_creds(self):
         except botocore.exceptions.ClientError as error:
             raise BespinError("Couldn't determine what account your credentials are from", error=error.message)
 
+        if self.session is None or self.session.region_name != self.region:
+            raise ProgrammerError("botocore.session created in incorrect region")
+
 
     def assume(self):
         assumed_role = "arn:aws:iam::{0}:{1}".format(self.account_id, self.assume_role)
@@ -58,24 +58,28 @@ def assume(self):
                 del os.environ[name]
 
         try:
-            conn = boto.sts.connect_to_region(self.region)
-        except boto.exception.NoAuthHandlerFound:
+            conn = boto3.client('sts', region_name=self.region)
+            session_name = "{1}@bespin{0}".format(VERSION, os.environ.get("USER", "<unknown_user>"))
+            response = conn.assume_role(RoleArn=assumed_role, RoleSessionName=session_name)
+
+            role = response['AssumedRoleUser']
+            creds = response['Credentials']
+            log.info("Assumed role (%s)", role['Arn'])
+            self.session = boto3.session.Session(
+                    aws_access_key_id=creds['AccessKeyId'],
+                    aws_secret_access_key=creds['SecretAccessKey'],
+                    aws_session_token=creds['SessionToken'],
+                    region_name=self.region
+            )
+
+            os.environ['AWS_ACCESS_KEY_ID'] = creds["AccessKeyId"]
+            os.environ['AWS_SECRET_ACCESS_KEY'] = creds["SecretAccessKey"]
+            os.environ['AWS_SECURITY_TOKEN'] = creds["SessionToken"]
+            os.environ['AWS_SESSION_TOKEN'] = creds["SessionToken"]
+        except botocore.exceptions.NoCredentialsError:
             raise BespinError("Export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY before running this script (your aws credentials)")
-
-        try:
-            creds = conn.assume_role(assumed_role, "bespin")
-        except boto.exception.BotoServerError as error:
-            if error.status == 403:
-                raise BespinError("Not allowed to assume role", error=error.message)
-            else:
-                raise
-
-        creds_dict = creds.credentials.to_dict()
-
-        os.environ['AWS_ACCESS_KEY_ID'] = creds_dict["access_key"]
-        os.environ['AWS_SECRET_ACCESS_KEY'] = creds_dict["secret_key"]
-        os.environ['AWS_SECURITY_TOKEN'] = creds_dict["session_token"]
-        os.environ['AWS_SESSION_TOKEN'] = creds_dict["session_token"]
+        except botocore.exceptions.ClientError as error:
+            raise BespinError("Unable to assume role", error=error.message)
 
     @memoized_property
     def s3(self):
@@ -101,7 +105,7 @@ def kms(self):
     def iam(self):
         self.verify_creds()
         log.info("Using region [%s] for iam", self.region)
-        return boto.iam.connect_to_region(self.region)
+        return self.session.client('iam', region_name=self.region)
 
     def cloudformation(self, stack_name):
         self.verify_creds()

bespin/amazon/mixin.py

@@ -2,29 +2,30 @@
 
 from contextlib import contextmanager
 import logging
-import boto
+import botocore
 
 log = logging.getLogger("bespin.amazon.mixin")
 
 class AmazonMixin(object):
     @contextmanager
     def catch_boto_400(self, errorkls, message, **info):
-        """Turn a BotoServerError 400 into a BadAmazon"""
+        """Turn a boto HTTP 400 into a BadAmazon"""
         try:
             yield
-        except boto.exception.BotoServerError as error:
-            if error.status == 400:
-                log.error("%s -(%s)- %s", message, error.code, error.message)
-                raise errorkls(message, error_code=error.code, error_message=error.message, **info)
+        except botocore.exceptions.ClientError as error:
+            code = error.response['ResponseMetadata']['HTTPStatusCode']
+            if code == 400:
+                log.error("%s -(%s)- %s", message, code, error.message)
+                raise errorkls(message, error_code=code, error_message=error.message, **info)
             else:
                 raise
 
     @contextmanager
     def ignore_throttling_error(self):
         try:
             yield
-        except boto.exception.BotoServerError as error:
-            if error.status == 400 and error.code == "Throttling":
+        except botocore.exceptions.ClientError as error:
+            if error.response['Error']['Code'] == 'Throttling':
                 raise Throttled()
             else:
                 raise

bespin/option_spec/stack_objs.py

@@ -252,6 +252,9 @@ def create_or_update(self):
         """Create or update the stack, return True if the stack actually changed"""
         log.info("Creating or updating the stack (%s)", self.stack_name)
         status = self.cloudformation.wait(may_not_exist=True)
+        tags = self.tags or None
+        if tags and type(tags) is not dict and hasattr(self.tags, "as_dict"):
+            tags = tags.as_dict()
 
         if not status.exists:
             log.info("No existing stack, making one now")
@@ -260,9 +263,6 @@ def create_or_update(self):
                 log.info("Would use following stack from {0}".format(self.stack_json))
                 print(self.dumped_stack_obj)
             else:
-                tags = self.tags or None
-                if tags and type(tags) is not dict and hasattr(self.tags, "as_dict"):
-                    tags = tags.as_dict()
                 return self.cloudformation.create(self.dumped_stack_obj, self.params_json_obj, tags)
         elif status.complete:
             log.info("Found existing stack, doing an update")
@@ -271,7 +271,7 @@ def create_or_update(self):
                 log.info("Would use following stack from {0}".format(self.stack_json))
                 print(json.dumps(self.dumped_stack_obj))
             else:
-                return self.cloudformation.update(self.dumped_stack_obj, self.params_json_obj)
+                return self.cloudformation.update(self.dumped_stack_obj, self.params_json_obj, tags)
         else:
             raise BadStack("Stack could not be updated", name=self.stack_name, status=status.name)