Merge pull request #74 from ministryofjustice/feature/extend-template

Cross account IAM deployment

CHANGELOG.md

@@ -1,3 +1,8 @@
+## Unreleased
+
+* Add include functionality for including extra cloudformation json files.
+* Add ability to use cross account IAM roles when authenticating to AWS.
+
 ## Version 0.2.1
 
 * Fix rsync missing passwords yaml from salt-pillar

README.rst

@@ -73,6 +73,15 @@ This tool needs AWS credentials to create stacks and the credentials should be p
     aws_access_key_id = AKIAI***********
     aws_secret_access_key = *******************************************
 
+If you wish to authenticate to a separate AWS account using cross account IAM roles you should create a profile called `cross-account` with the access keys of the user with permission to assume roles from the second account::
+
+    [cross-account]
+    aws_access_key_id = AKIAI***********
+    aws_secret_access_key = *******************************************
+
+And when you run the tool you must set the ARN ID of the role in the separate account which you wish to assume. For example::
+
+    AWS_ROLE_ARN_ID='arn:aws:iam::123456789012:role/S3Access' fab application:courtfinder aws:prod environment:dev config:/path/to/courtfinder-dev.yaml cfn_create
 
 Project specific YAML file
 +++++++++++++++++++++++++++
@@ -120,3 +129,11 @@ You can add a custom s3 policy to override bootstrap-cfn's default settings. For
      static-bucket-name: moj-test-dev-static
      policy: tests/sample-custom-s3-policy.json
 
+Includes
+++++++++++
+If you wish to include some static cloudformation json and have it merged with the template generated by bootstrap-cfn. You can do the following in your template yaml file::
+
+    includes:
+      - /path/to/cloudformation.json
+
+The tool will then perform a deep merge of the includes with the generated template dictionary. Any keys or subkeys in the template dictionary that clash will have their values **overwritten** by the included dictionary or recursively merged if the value is itself a dictionary.

bootstrap_cfn/config.py

@@ -4,6 +4,8 @@
 import sys
 import yaml
 import bootstrap_cfn.errors as errors
+import bootstrap_cfn.utils as utils
+
 from copy import deepcopy
 
 class ProjectConfig:
@@ -14,31 +16,13 @@ def __init__(self, config, environment, passwords=None):
         self.config = self.load_yaml(config)[environment]
         if passwords:
             passwords_dict = self.load_yaml(passwords)[environment]
-            self.config = self.dict_merge(self.config, passwords_dict)
+            self.config = utils.dict_merge(self.config, passwords_dict)
 
     @staticmethod
     def load_yaml(fp):
         if os.path.exists(fp):
             return yaml.load(open(fp).read())
 
-    def dict_merge(self, target, *args):
-        # Merge multiple dicts
-        if len(args) > 1:
-            for obj in args:
-                self.dict_merge(target, obj)
-            return target
-
-        # Recursively merge dicts and set non-dict values
-        obj = args[0]
-        if not isinstance(obj, dict):
-            return obj
-        for k, v in obj.iteritems():
-            if k in target and isinstance(target[k], dict):
-                self.dict_merge(target[k], v)
-            else:
-                target[k] = deepcopy(v)
-        return target
-
 
 class ConfigParser:
 
@@ -92,6 +76,10 @@ def process(self):
         template['Outputs'] = {}
         for t in output_templates:
             template['Outputs'].update(json.loads(pkgutil.get_data('bootstrap_cfn', t)))
+        if 'includes' in self.data:
+            for inc_path in self.data['includes']:
+                inc = json.load(open(inc_path))
+                template = utils.dict_merge(template, inc)
         return json.dumps(
             template, sort_keys=True, indent=4, separators=(',', ': '))
 

bootstrap_cfn/salt_utils.py

@@ -1,12 +1,17 @@
 #!/usr/bin/env python
+import sys
+import os
+#We adjust the path here so that this script can be run
+#by a user without changing their PYTHONPATH, useful if they
+#have limited sudo access to run the script.
+sys.path.append(os.path.dirname(__file__) + '/..')
 from bootstrap_cfn import utils
 from bootstrap_cfn import errors
 import salt
 import salt.runner
 import salt.client
 import pprint
 import time
-import sys
 
 def start_highstate(target):
     local = salt.client.LocalClient()

bootstrap_cfn/utils.py

@@ -1,9 +1,12 @@
 import boto.exception
 import boto.provider
+import boto.sts
 import sys
 import time
+import os
 
 import bootstrap_cfn.errors as errors
+from copy import deepcopy
 
 
 def timeout(timeout, interval):
@@ -24,6 +27,22 @@ def wrapper(*args, **kwargs):
 
 def connect_to_aws(module, instance):
     try:
+        if instance.aws_profile_name == 'cross-account':
+            sts = boto.sts.connect_to_region(
+                region_name=instance.aws_region_name,
+                profile_name=instance.aws_profile_name
+            )
+            role = sts.assume_role(
+                role_arn=os.environ['AWS_ROLE_ARN_ID'],
+                role_session_name="AssumeRoleSession1"
+            )
+            conn = module.connect_to_region(
+                region_name=instance.aws_region_name,
+                aws_access_key_id=role.credentials.access_key,
+                aws_secret_access_key=role.credentials.secret_key,
+                security_token=role.credentials.session_token
+            )
+            return conn
         conn = module.connect_to_region(
             region_name=instance.aws_region_name,
             profile_name=instance.aws_profile_name
@@ -33,3 +52,21 @@ def connect_to_aws(module, instance):
         raise errors.NoCredentialsError()
     except boto.provider.ProfileNotFoundError as e:
         raise errors.ProfileNotFoundError(instance.aws_profile_name)
+
+def dict_merge(target, *args):
+    # Merge multiple dicts
+    if len(args) > 1:
+        for obj in args:
+            dict_merge(target, obj)
+        return target
+
+    # Recursively merge dicts and set non-dict values
+    obj = args[0]
+    if not isinstance(obj, dict):
+        return obj
+    for k, v in obj.iteritems():
+        if k in target and isinstance(target[k], dict):
+            dict_merge(target[k], v)
+        else:
+            target[k] = deepcopy(v)
+    return target

tests/sample-include.json

@@ -0,0 +1,9 @@
+{
+  "Outputs":
+    {
+      "someoutput": {
+        "Description": "For tests",
+        "Value": "BLAHBLAH"
+      }
+   }
+}

tests/tests.py

@@ -4,6 +4,7 @@
 from bootstrap_cfn.config import ProjectConfig, ConfigParser
 import bootstrap_cfn.errors as errors
 from testfixtures import compare
+import json
 
 
 class TestConfig(unittest.TestCase):
@@ -339,6 +340,30 @@ def test_elb_custom_sg(self):
         compare(elb['Properties']['SecurityGroups'],
                 [{u'Ref': u'SGName'}])
 
+    def test_cf_includes(self):
+        project_config = ProjectConfig('tests/sample-project.yaml',
+                                       'dev', 
+                                       'tests/sample-project-passwords.yaml')
+        project_config.config['includes'] = ['tests/sample-include.json']
+        known_outputs = {
+          "dbhost": {
+            "Description": "RDS Hostname",
+            "Value": {"Fn::GetAtt" : [ "RDSInstance" , "Endpoint.Address" ]}
+          },
+          "dbport": {
+            "Description": "RDS Port",
+            "Value": {"Fn::GetAtt" : [ "RDSInstance" , "Endpoint.Port" ]}
+          },
+          "someoutput":{
+            "Description": "For tests",
+            "Value": "BLAHBLAH"
+          }
+        }
+        config = ConfigParser(project_config.config, 'my-stack-name')
+        cfg = json.loads(config.process())
+        outputs = cfg['Outputs']
+        compare(known_outputs, outputs)
+
     def test_process_no_elbs_no_rds(self):
         project_config = ProjectConfig('tests/sample-project.yaml', 'dev')
         # Assuming there's no ELB defined