Merge 057c036 into edb30a6

formica/change_set.py

@@ -1,6 +1,8 @@
 import sys
 
 import logging
+import uuid
+from formica.aws import AWS
 from botocore.exceptions import ClientError, WaiterError
 from texttable import Texttable
 
@@ -17,7 +19,7 @@ def __init__(self, stack, client):
         self.stack = stack
         self.client = client
 
-    def create(self, template, change_set_type, parameters=[], tags=[], capabilities=[], role_arn=None):
+    def create(self, template, change_set_type, parameters=[], tags=[], capabilities=[], role_arn=None, s3=False):
         optional_arguments = {}
         if parameters:
             optional_arguments['Parameters'] = [
@@ -33,19 +35,43 @@ def create(self, template, change_set_type, parameters=[], tags=[], capabilities
         if change_set_type == 'UPDATE':
             self.remove_existing_changeset()
 
-        self.client.create_change_set(StackName=self.stack, TemplateBody=template,
-                                      ChangeSetName=self.name, ChangeSetType=change_set_type,
-                                      **optional_arguments)
-        logger.info('Change set submitted, waiting for CloudFormation to calculate changes ...')
-        waiter = self.client.get_waiter('change_set_create_complete')
         try:
+            if s3:
+                session = AWS.current_session()
+                s3_client = session.client('s3')
+                bucket_name = 'formica-deploy-{}'.format(str(uuid.uuid4()).lower())
+                bucket_path = '{}-template.json'.format(self.stack)
+                logger.info('Creating Bucket: {}'.format(bucket_name))
+
+                s3_client.create_bucket(
+                    Bucket=bucket_name,
+                    CreateBucketConfiguration=dict(LocationConstraint=session.region_name)
+                )
+
+                logger.info('Uploading to bucket: {}/{}'.format(bucket_name, bucket_path))
+                s3_client.put_object(Bucket=bucket_name, Key=bucket_path, Body=template)
+                template_url = 'https://{}.s3.amazonaws.com/{}'.format(bucket_name, bucket_path)
+                optional_arguments['TemplateURL'] = template_url
+            else:
+                optional_arguments['TemplateBody'] = template
+
+            self.client.create_change_set(StackName=self.stack,
+                                          ChangeSetName=self.name, ChangeSetType=change_set_type,
+                                          **optional_arguments)
+            logger.info('Change set submitted, waiting for CloudFormation to calculate changes ...')
+            waiter = self.client.get_waiter('change_set_create_complete')
             waiter.wait(ChangeSetName=self.name, StackName=self.stack)
             logger.info('Change set created successfully')
         except WaiterError as e:
             status_reason = e.last_response.get('StatusReason', '')
             logger.info(status_reason)
             if "didn't contain changes" not in status_reason:
                 sys.exit(1)
+        finally:
+            if s3:
+                logger.info('Deleting Object and Bucket: {}/{}'.format(bucket_name, bucket_path))
+                s3_client.delete_object(Bucket=bucket_name, Key=bucket_path)
+                s3_client.delete_bucket(Bucket=bucket_name)
 
     def describe(self):
         change_set = self.client.describe_change_set(StackName=self.stack, ChangeSetName=self.name)

formica/cli.py

@@ -80,6 +80,7 @@ def main(cli_args):
     add_role_arn_argument(new_parser)
     add_config_file_argument(new_parser)
     add_stack_variables_argument(new_parser)
+    add_s3_upload_argument(new_parser)
     new_parser.set_defaults(func=new)
 
     # Change Command Arguments
@@ -92,6 +93,7 @@ def main(cli_args):
     add_role_arn_argument(change_parser)
     add_config_file_argument(change_parser)
     add_stack_variables_argument(change_parser)
+    add_s3_upload_argument(change_parser)
     change_parser.set_defaults(func=change)
 
     # Deploy Command Arguments
@@ -211,6 +213,10 @@ def add_config_file_argument(parser):
                         nargs='+')
 
 
+def add_s3_upload_argument(parser):
+    parser.add_argument('--s3', help='Upload template to S3 before deployment', action='store_true')
+
+
 def template(args):
     loader = Loader(variables=args.vars)
     loader.load()
@@ -281,7 +287,7 @@ def change(args):
 
     change_set = ChangeSet(stack=args.stack, client=client)
     change_set.create(template=loader.template(indent=None), change_set_type='UPDATE', parameters=args.parameters,
-                      tags=args.tags, capabilities=args.capabilities, role_arn=args.role_arn)
+                      tags=args.tags, capabilities=args.capabilities, role_arn=args.role_arn, s3=args.s3)
     change_set.describe()
 
 
@@ -314,7 +320,7 @@ def new(args):
     logger.info('Creating change set for new stack, ...')
     change_set = ChangeSet(stack=args.stack, client=client)
     change_set.create(template=loader.template(indent=None), change_set_type='CREATE', parameters=args.parameters,
-                      tags=args.tags, capabilities=args.capabilities, role_arn=args.role_arn)
+                      tags=args.tags, capabilities=args.capabilities, role_arn=args.role_arn, s3=args.s3)
     change_set.describe()
     logger.info('Change set created, please deploy')
 

stacks/test.config.yaml

@@ -1,5 +1,5 @@
 stack: formica-test-stack
-role-arn: arn:aws:iam::080551076419:role/test/1/formica-test-stack-TestRole1-NPMDYV78VC8J
+# role-arn: arn:aws:iam::080551076419:role/test/1/formica-test-stack-2-TestRole1-1PGVZJ88QDVBD
 capabilities:
   - CAPABILITY_IAM
 vars:

stacks/test.template.yaml

@@ -31,6 +31,6 @@ Resources:
   Bucket1:
     From: Modules
     Properties:
-      BucketName: TestName
+      BucketName: formica-test-bucket-name-something-here
   Bucket2:
     From: Modules::Submodule::Submodule

tests/integration/test_basic.py

@@ -57,7 +57,7 @@ def write_config(content):
             assert 'Dictionary Item Added' in diff
 
             # Change Resources in existing stack
-            change = run_formica('change', *stack_args)
+            change = run_formica('change', '--s3', *stack_args)
             assert 'TestNameUpdate' in change
 
             # Describe ChangeSet before deploying

tests/unit/constants.py

@@ -1,4 +1,5 @@
 import datetime
+import uuid
 
 from dateutil.tz import tzlocal
 
@@ -14,6 +15,7 @@
 CHANGE_SET_STACK_TAGS = {'A': 'B', 'B': 'C'}
 CHANGE_SET_CAPABILITIES = ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']
 VARS = {'SomeVar': 'value', 'OtherVar': 2}
+UUID = str(uuid.uuid4())
 CHANGESETCHANGES = {'ChangeSetName': 'simpleteststack-change-set', 'ChangeSetId': 'arn:aws:cloudformation:eu-central-1:420759548424:changeSet/simpleteststack-change-set/979f29ac-40c9-4802-b496-0b3f38241bcd', 'StackId': 'arn:aws:cloudformation:eu-central-1:420759548424:stack/simpleteststack/a4f23770-e476-11e6-bfa4-500c44f62262', 'StackName': 'simpleteststack', 'Parameters': [{'ParameterKey': 'bucketname', 'ParameterValue': 'formicatestbucketname'}, {'ParameterKey': 'bucketname2', 'ParameterValue': 'formicatestbucketname2'}], 'CreationTime': datetime.datetime(2017, 1, 27, 9, 58, 3, 821000, tzinfo=tzlocal()), 'ExecutionStatus': 'AVAILABLE', 'Status': 'CREATE_COMPLETE', 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], 'Tags': [{'Key': 'StackKey', 'Value': 'StackValue'}, {'Key': 'StackKey2', 'Value': 'StackValue2'}], 'Changes': [{'Type': 'Resource', 'ResourceChange': {'Action': 'Remove', 'LogicalResourceId': 'DeploymentBucket', 'PhysicalResourceId': 'simpleteststack-deploymentbucket-1l7p61v6fxpry', 'ResourceType': 'AWS::S3::Bucket', 'Scope': [], 'Details': []}}, {'Type': 'Resource', 'ResourceChange': {'Action': 'Modify', 'LogicalResourceId': 'DeploymentBucket2', 'PhysicalResourceId': 'simpleteststack-deploymentbucket2-11ngaeftydtn7', 'ResourceType': 'AWS::S3::Bucket', 'Replacement': 'True', 'Scope': ['Properties', 'Tags'], 'Details': [{'Target': {'Attribute': 'Tags', 'RequiresRecreation': 'Never'}, 'Evaluation': 'Static', 'ChangeSource': 'DirectModification'}, {'Target': {'Attribute': 'Properties', 'Name': 'BucketName', 'RequiresRecreation': 'Always'}, 'Evaluation': 'Static', 'ChangeSource': 'DirectModification'}]}}, {'Type': 'Resource', 'ResourceChange': {'Action': 'Add', 'LogicalResourceId': 'DeploymentBucket3', 'ResourceType': 'AWS::S3::Bucket', 'Scope': [], 'Details': []}}], 'ResponseMetadata': {'RequestId': '2a95cc4f-e477-11e6-a696-5fdc4c9bb8c5', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '2a95cc4f-e477-11e6-a696-5fdc4c9bb8c5', 'content-type': 'text/xml', 'content-length': '2816', 'vary': 'Accept-Encoding', 'date': 'Fri, 27 Jan 2017 09:58:33 GMT'}, 'RetryAttempts': 0}}  # noqa
 CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER = {'ChangeSetName': 'simpleteststack-change-set', 'ChangeSetId': 'arn:aws:cloudformation:eu-central-1:420759548424:changeSet/simpleteststack-change-set/979f29ac-40c9-4802-b496-0b3f38241bcd', 'StackId': 'arn:aws:cloudformation:eu-central-1:420759548424:stack/simpleteststack/a4f23770-e476-11e6-bfa4-500c44f62262', 'StackName': 'simpleteststack', 'CreationTime': datetime.datetime(2017, 1, 27, 9, 58, 3, 821000, tzinfo=tzlocal()), 'ExecutionStatus': 'AVAILABLE', 'Status': 'CREATE_COMPLETE', 'NotificationARNs': [], 'Capabilities': [], 'Changes': [{'Type': 'Resource', 'ResourceChange': {'Action': 'Remove', 'LogicalResourceId': 'DeploymentBucket', 'PhysicalResourceId': 'simpleteststack-deploymentbucket-1l7p61v6fxpry', 'ResourceType': 'AWS::S3::Bucket', 'Scope': [], 'Details': []}}, {'Type': 'Resource', 'ResourceChange': {'Action': 'Modify', 'LogicalResourceId': 'DeploymentBucket2', 'PhysicalResourceId': 'simpleteststack-deploymentbucket2-11ngaeftydtn7', 'ResourceType': 'AWS::S3::Bucket', 'Replacement': 'True', 'Scope': ['Properties', 'Tags'], 'Details': [{'Target': {'Attribute': 'Tags', 'RequiresRecreation': 'Never'}, 'Evaluation': 'Static', 'ChangeSource': 'DirectModification'}, {'Target': {'Attribute': 'Properties', 'Name': 'BucketName', 'RequiresRecreation': 'Always'}, 'Evaluation': 'Static', 'ChangeSource': 'DirectModification'}, {'Target': {'Attribute': 'Properties', 'Name': 'BucketName', 'RequiresRecreation': 'Always'}, 'Evaluation': 'Static', 'ChangeSource': 'DirectModification'}]}}, {'Type': 'Resource', 'ResourceChange': {'Action': 'Add', 'LogicalResourceId': 'DeploymentBucket3', 'ResourceType': 'AWS::S3::Bucket', 'Scope': [], 'Details': []}}], 'ResponseMetadata': {'RequestId': '2a95cc4f-e477-11e6-a696-5fdc4c9bb8c5', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '2a95cc4f-e477-11e6-a696-5fdc4c9bb8c5', 'content-type': 'text/xml', 'content-length': '2816', 'vary': 'Accept-Encoding', 'date': 'Fri, 27 Jan 2017 09:58:33 GMT'}, 'RetryAttempts': 0}}  # noqa
 

tests/unit/test_change.py

@@ -14,32 +14,36 @@ def session(mocker):
     return mocker.patch('formica.aws.Session')
 
 
+@pytest.fixture
+def client(session):
+    client_mock = Mock()
+    session.return_value.client.return_value = client_mock
+    return client_mock
+
+
 @pytest.fixture
 def loader(mocker):
     return mocker.patch('formica.cli.Loader')
 
 
-def test_change_creates_update_change_set(change_set, session, loader):
-    client_mock = Mock()
-    session.return_value.client.return_value = client_mock
+def test_change_creates_update_change_set(change_set, client, loader):
+    print(client)
     loader.return_value.template.return_value = TEMPLATE
     cli.main(['change', '--stack', STACK, '--profile', PROFILE, '--region', REGION])
-    change_set.assert_called_with(stack=STACK, client=client_mock)
+    change_set.assert_called_with(stack=STACK, client=client)
     change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='UPDATE',
                                                            parameters=None,
-                                                           tags=None, capabilities=None, role_arn=None)
+                                                           tags=None, capabilities=None, role_arn=None, s3=False)
     change_set.return_value.describe.assert_called_once()
 
 
-def test_change_uses_parameters_for_update(change_set, session, loader):
-    client_mock = Mock()
-    session.return_value.client.return_value = client_mock
+def test_change_uses_parameters_for_update(change_set, client, loader):
     loader.return_value.template.return_value = TEMPLATE
     cli.main(['change', '--stack', STACK, '--parameters', 'A=B', 'C=D', '--profile', PROFILE, '--region', REGION])
-    change_set.assert_called_with(stack=STACK, client=client_mock)
+    change_set.assert_called_with(stack=STACK, client=client)
     change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='UPDATE',
                                                            parameters={'A': 'B', 'C': 'D'}, tags=None,
-                                                           capabilities=None, role_arn=None)
+                                                           capabilities=None, role_arn=None, s3=False)
     change_set.return_value.describe.assert_called_once()
 
 
@@ -51,15 +55,13 @@ def test_change_tests_parameter_format(capsys):
     assert pytest_wrapped_e.value.code == 2
 
 
-def test_change_uses_tags_for_creation(change_set, session, loader):
-    client_mock = Mock()
-    session.return_value.client.return_value = client_mock
+def test_change_uses_tags_for_creation(change_set, client, loader):
     loader.return_value.template.return_value = TEMPLATE
     cli.main(['change', '--stack', STACK, '--tags', 'A=B', 'C=D', '--profile', PROFILE, '--region', REGION])
-    change_set.assert_called_with(stack=STACK, client=client_mock)
+    change_set.assert_called_with(stack=STACK, client=client)
     change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='UPDATE',
-                                                           parameters=None,
-                                                           tags={'A': 'B', 'C': 'D'}, capabilities=None, role_arn=None)
+                                                           parameters=None, tags={'A': 'B', 'C': 'D'},
+                                                           capabilities=None, role_arn=None, s3=False)
 
 
 def test_change_tests_tag_format(capsys):
@@ -71,12 +73,19 @@ def test_change_tests_tag_format(capsys):
     assert pytest_wrapped_e.value.code == 2
 
 
-def test_change_uses_capabilities_for_creation(change_set, session, loader):
-    client_mock = Mock()
-    session.return_value.client.return_value = client_mock
+def test_change_uses_capabilities_for_creation(change_set, client, loader):
     loader.return_value.template.return_value = TEMPLATE
     cli.main(['change', '--stack', STACK, '--capabilities', 'A', 'B'])
-    change_set.assert_called_with(stack=STACK, client=client_mock)
+    change_set.assert_called_with(stack=STACK, client=client)
+    change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='UPDATE',
+                                                           parameters=None,
+                                                           tags=None, capabilities=['A', 'B'], role_arn=None, s3=False)
+
+
+def test_change_sets_s3_flag(change_set, client, loader):
+    loader.return_value.template.return_value = TEMPLATE
+    cli.main(['change', '--stack', STACK, '--s3'])
+    change_set.assert_called_with(stack=STACK, client=client)
     change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='UPDATE',
                                                            parameters=None,
-                                                           tags=None, capabilities=['A', 'B'], role_arn=None)
+                                                           tags=None, capabilities=None, role_arn=None, s3=True)

tests/unit/test_change_set.py

@@ -7,7 +7,7 @@
 from tests.unit.constants import (
     STACK, TEMPLATE, CHANGE_SET_TYPE, CHANGESETNAME, CHANGESETCHANGES,
     CHANGE_SET_PARAMETERS, ROLE_ARN, CHANGE_SET_STACK_TAGS,
-    CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER,
+    CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER, REGION, UUID
 )
 
 
@@ -16,6 +16,22 @@ def logger(mocker):
     return mocker.patch('formica.change_set.logger')
 
 
+@pytest.fixture
+def client(mocker):
+    AWS = mocker.patch('formica.change_set.AWS')
+    client_mock = mocker.Mock()
+    AWS.current_session.return_value.client.return_value = client_mock
+    AWS.current_session.return_value.region_name = REGION
+    return client_mock
+
+
+@pytest.fixture
+def uuid(mocker):
+    uuid = mocker.patch('formica.change_set.uuid')
+    uuid.uuid4.return_value = UUID
+    return uuid
+
+
 def test_submits_changeset_and_waits():
     cf_client_mock = Mock()
     change_set = ChangeSet(STACK, cf_client_mock)
@@ -32,6 +48,26 @@ def test_submits_changeset_and_waits():
         StackName=STACK, ChangeSetName=CHANGESETNAME)
 
 
+def test_creates_and_removes_bucket_for_s3_flag(client, uuid):
+    change_set = ChangeSet(STACK, client)
+
+    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, s3=True)
+    bucket_name = 'formica-deploy-{}'.format(UUID)
+    bucket_path = '{}-template.json'.format(STACK)
+    template_url = 'https://{}.s3.amazonaws.com/{}'.format(bucket_name, bucket_path)
+
+    client.create_bucket.assert_called_with(Bucket=bucket_name,
+                                            CreateBucketConfiguration=dict(LocationConstraint=REGION))
+    client.put_object.assert_called_with(Bucket=bucket_name, Key=bucket_path, Body=TEMPLATE)
+
+    client.create_change_set.assert_called_with(
+        StackName=STACK, TemplateURL=template_url,
+        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE)
+
+    client.delete_object.assert_called_with(Bucket=bucket_name, Key=bucket_path)
+    client.delete_bucket.assert_called_with(Bucket=bucket_name)
+
+
 def test_submits_changeset_with_parameters():
     cf_client_mock = Mock()
     change_set = ChangeSet(STACK, cf_client_mock)