diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c40ae4..ff78239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased + +* Add include functionality for including extra cloudformation json files. +* Add ability to use cross account IAM roles when authenticating to AWS. + ## Version 0.2.1 * Fix rsync missing passwords yaml from salt-pillar diff --git a/README.rst b/README.rst index f243f34..916cb8c 100644 --- a/README.rst +++ b/README.rst @@ -73,6 +73,15 @@ This tool needs AWS credentials to create stacks and the credentials should be p aws_access_key_id = AKIAI*********** aws_secret_access_key = ******************************************* +If you wish to authenticate to a separate AWS account using cross account IAM roles you should create a profile called `cross-account` with the access keys of the user with permission to assume roles from the second account:: + + [cross-account] + aws_access_key_id = AKIAI*********** + aws_secret_access_key = ******************************************* + +And when you run the tool you must set the ARN ID of the role in the separate account which you wish to assume. For example:: + + AWS_ROLE_ARN_ID='arn:aws:iam::123456789012:role/S3Access' fab application:courtfinder aws:prod environment:dev config:/path/to/courtfinder-dev.yaml cfn_create Project specific YAML file +++++++++++++++++++++++++++ @@ -120,3 +129,11 @@ You can add a custom s3 policy to override bootstrap-cfn's default settings. For static-bucket-name: moj-test-dev-static policy: tests/sample-custom-s3-policy.json +Includes +++++++++++ +If you wish to include some static cloudformation json and have it merged with the template generated by bootstrap-cfn. You can do the following in your template yaml file:: + + includes: + - /path/to/cloudformation.json + +The tool will then perform a deep merge of the includes with the generated template dictionary. Any keys or subkeys in the template dictionary that clash will have their values **overwritten** by the included dictionary or recursively merged if the value is itself a dictionary. diff --git a/bootstrap_cfn/config.py b/bootstrap_cfn/config.py index ee4b125..718ffdc 100644 --- a/bootstrap_cfn/config.py +++ b/bootstrap_cfn/config.py @@ -4,6 +4,8 @@ import sys import yaml import bootstrap_cfn.errors as errors +import bootstrap_cfn.utils as utils + from copy import deepcopy class ProjectConfig: @@ -14,31 +16,13 @@ def __init__(self, config, environment, passwords=None): self.config = self.load_yaml(config)[environment] if passwords: passwords_dict = self.load_yaml(passwords)[environment] - self.config = self.dict_merge(self.config, passwords_dict) + self.config = utils.dict_merge(self.config, passwords_dict) @staticmethod def load_yaml(fp): if os.path.exists(fp): return yaml.load(open(fp).read()) - def dict_merge(self, target, *args): - # Merge multiple dicts - if len(args) > 1: - for obj in args: - self.dict_merge(target, obj) - return target - - # Recursively merge dicts and set non-dict values - obj = args[0] - if not isinstance(obj, dict): - return obj - for k, v in obj.iteritems(): - if k in target and isinstance(target[k], dict): - self.dict_merge(target[k], v) - else: - target[k] = deepcopy(v) - return target - class ConfigParser: @@ -92,6 +76,10 @@ def process(self): template['Outputs'] = {} for t in output_templates: template['Outputs'].update(json.loads(pkgutil.get_data('bootstrap_cfn', t))) + if 'includes' in self.data: + for inc_path in self.data['includes']: + inc = json.load(open(inc_path)) + template = utils.dict_merge(template, inc) return json.dumps( template, sort_keys=True, indent=4, separators=(',', ': ')) diff --git a/bootstrap_cfn/salt_utils.py b/bootstrap_cfn/salt_utils.py index 09d45ef..41d78f4 100644 --- a/bootstrap_cfn/salt_utils.py +++ b/bootstrap_cfn/salt_utils.py @@ -1,4 +1,10 @@ #!/usr/bin/env python +import sys +import os +#We adjust the path here so that this script can be run +#by a user without changing their PYTHONPATH, useful if they +#have limited sudo access to run the script. +sys.path.append(os.path.dirname(__file__) + '/..') from bootstrap_cfn import utils from bootstrap_cfn import errors import salt @@ -6,7 +12,6 @@ import salt.client import pprint import time -import sys def start_highstate(target): local = salt.client.LocalClient() diff --git a/bootstrap_cfn/utils.py b/bootstrap_cfn/utils.py index f9720d1..981676d 100644 --- a/bootstrap_cfn/utils.py +++ b/bootstrap_cfn/utils.py @@ -1,9 +1,12 @@ import boto.exception import boto.provider +import boto.sts import sys import time +import os import bootstrap_cfn.errors as errors +from copy import deepcopy def timeout(timeout, interval): @@ -24,6 +27,22 @@ def wrapper(*args, **kwargs): def connect_to_aws(module, instance): try: + if instance.aws_profile_name == 'cross-account': + sts = boto.sts.connect_to_region( + region_name=instance.aws_region_name, + profile_name=instance.aws_profile_name + ) + role = sts.assume_role( + role_arn=os.environ['AWS_ROLE_ARN_ID'], + role_session_name="AssumeRoleSession1" + ) + conn = module.connect_to_region( + region_name=instance.aws_region_name, + aws_access_key_id=role.credentials.access_key, + aws_secret_access_key=role.credentials.secret_key, + security_token=role.credentials.session_token + ) + return conn conn = module.connect_to_region( region_name=instance.aws_region_name, profile_name=instance.aws_profile_name @@ -33,3 +52,21 @@ def connect_to_aws(module, instance): raise errors.NoCredentialsError() except boto.provider.ProfileNotFoundError as e: raise errors.ProfileNotFoundError(instance.aws_profile_name) + +def dict_merge(target, *args): + # Merge multiple dicts + if len(args) > 1: + for obj in args: + dict_merge(target, obj) + return target + + # Recursively merge dicts and set non-dict values + obj = args[0] + if not isinstance(obj, dict): + return obj + for k, v in obj.iteritems(): + if k in target and isinstance(target[k], dict): + dict_merge(target[k], v) + else: + target[k] = deepcopy(v) + return target diff --git a/tests/sample-include.json b/tests/sample-include.json new file mode 100644 index 0000000..f59314d --- /dev/null +++ b/tests/sample-include.json @@ -0,0 +1,9 @@ +{ + "Outputs": + { + "someoutput": { + "Description": "For tests", + "Value": "BLAHBLAH" + } + } +} diff --git a/tests/tests.py b/tests/tests.py index 129291c..a752a14 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -4,6 +4,7 @@ from bootstrap_cfn.config import ProjectConfig, ConfigParser import bootstrap_cfn.errors as errors from testfixtures import compare +import json class TestConfig(unittest.TestCase): @@ -339,6 +340,30 @@ def test_elb_custom_sg(self): compare(elb['Properties']['SecurityGroups'], [{u'Ref': u'SGName'}]) + def test_cf_includes(self): + project_config = ProjectConfig('tests/sample-project.yaml', + 'dev', + 'tests/sample-project-passwords.yaml') + project_config.config['includes'] = ['tests/sample-include.json'] + known_outputs = { + "dbhost": { + "Description": "RDS Hostname", + "Value": {"Fn::GetAtt" : [ "RDSInstance" , "Endpoint.Address" ]} + }, + "dbport": { + "Description": "RDS Port", + "Value": {"Fn::GetAtt" : [ "RDSInstance" , "Endpoint.Port" ]} + }, + "someoutput":{ + "Description": "For tests", + "Value": "BLAHBLAH" + } + } + config = ConfigParser(project_config.config, 'my-stack-name') + cfg = json.loads(config.process()) + outputs = cfg['Outputs'] + compare(known_outputs, outputs) + def test_process_no_elbs_no_rds(self): project_config = ProjectConfig('tests/sample-project.yaml', 'dev') # Assuming there's no ELB defined