From f307e5df500b274db22ab09bf25bcaa43607d137 Mon Sep 17 00:00:00 2001 From: Troy Ready Date: Mon, 2 Oct 2017 14:26:07 -0700 Subject: [PATCH] add remote config support (#458) * add remote config support Extends package_sources to support remote configuration files. * update docs about hook dictionaries * cleanup for code review * additional cleanups * Clean up the dict -> list stuff --- docs/config.rst | 37 +++++++- stacker/config/__init__.py | 68 +++++++++++++-- stacker/tests/test_config.py | 158 +++++++++++++++++++++++++---------- stacker/tests/test_util.py | 72 ++++++++++++++-- stacker/util.py | 120 ++++++++++++++++---------- 5 files changed, 354 insertions(+), 101 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index d575e1c0e..324fa6cf4 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -146,6 +146,42 @@ Cloned repositories will be cached between builds; the cache location defaults to ~/.stacker but can be manually specified via the **stacker_cache_dir** top level keyword. +Remote Configs +~~~~~~~~~~~~~~ +Configuration yamls from remote configs can also be used by specifying a list +of ``configs`` in the repo to use:: + + package_sources: + git: + - uri: git@github.com:acmecorp/stacker_blueprints.git + configs: + - vpc.yaml + +In this example, the configuration in ``vpc.yaml`` will be merged into the +running current configuration, with the current configuration's values taking +priority over the values in ``vpc.yaml``. + +Dictionary Stack Names & Hook Paths +::::::::::::::::::::::::::::::::::: +To allow remote configs to be selectively overriden, stack names & hook +paths can optionally be defined as dictionaries, e.g.:: + + pre_build: + my_route53_hook: + path: stacker.hooks.route53.create_domain: + required: true + args: + domain: mydomain.com + stacks: + vpc-example: + class_path: stacker_blueprints.vpc.VPC + locked: false + enabled: true + bastion-example: + class_path: stacker_blueprints.bastion.Bastion + locked: false + enabled: true + Pre & Post Hooks ---------------- @@ -316,7 +352,6 @@ Here's an example from stacker_blueprints_, used to create a VPC:: - 10.128.20.0/22 CidrBlock: 10.128.0.0/16 - Variables ========== diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index 1feffcbe1..bc4c74786 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -1,3 +1,4 @@ +import copy import sys import logging @@ -19,7 +20,7 @@ import yaml from ..lookups import register_lookup_handler -from ..util import SourceProcessor +from ..util import merge_map, yaml_to_ordered_dict, SourceProcessor from .. import exceptions # register translators (yaml constructors) @@ -43,7 +44,9 @@ def render_parse_load(raw_config, environment=None, validate=True): """ - rendered = render(raw_config, environment) + pre_rendered = render(raw_config, environment) + + rendered = process_remote_sources(pre_rendered, environment) config = parse(rendered) @@ -106,6 +109,23 @@ def parse(raw_config): """ + # Convert any applicable dictionaries back into lists + # This is necessary due to the move from lists for these top level config + # values to either lists or OrderedDicts. + # Eventually we should probably just make them OrderedDicts only. + config_dict = yaml_to_ordered_dict(raw_config) + for top_level_key in ['stacks', 'pre_build', 'post_build', 'pre_destroy', + 'post_destroy']: + top_level_value = config_dict.get(top_level_key) + if isinstance(top_level_value, dict): + tmp_list = [] + for key, value in top_level_value.iteritems(): + tmp_dict = copy.deepcopy(value) + if top_level_key == 'stacks': + tmp_dict['name'] = key + tmp_list.append(tmp_dict) + config_dict[top_level_key] = tmp_list + # We have to enable non-strict mode, because people may be including top # level keys for re-use with stacks (e.g. including something like # `common_variables: &common_variables`). @@ -118,7 +138,7 @@ def parse(raw_config): # should consider enabling this in the future. strict = False - return Config(yaml.safe_load(raw_config), strict=strict) + return Config(config_dict, strict=strict) def load(config): @@ -140,12 +160,6 @@ def load(config): if config.lookups: for key, handler in config.lookups.iteritems(): register_lookup_handler(key, handler) - sources = config.package_sources - if sources is not None: - processor = SourceProcessor( - stacker_cache_dir=config.stacker_cache_dir - ) - processor.get_package_sources(sources=sources) return config @@ -169,6 +183,40 @@ def dump(config): allow_unicode=True) +def process_remote_sources(raw_config, environment=None): + """Stage remote package sources and merge in remote configs. + + Args: + raw_config (str): the raw stacker configuration string. + environment (dict, optional): any environment values that should be + passed to the config + + Returns: + str: the raw stacker configuration string + + """ + + config = yaml.safe_load(raw_config) + if config.get('package_sources'): + processor = SourceProcessor( + sources=config['package_sources'], + stacker_cache_dir=config.get('stacker_cache_dir') + ) + processor.get_package_sources() + if processor.configs_to_merge: + for i in processor.configs_to_merge: + logger.debug("Merging in remote config \"%s\"", i) + remote_config = yaml.safe_load(open(i)) + config = merge_map(remote_config, config) + # Call the render again as the package_sources may have merged in + # additional environment lookups + if not environment: + environment = {} + return render(str(config), environment) + + return raw_config + + def not_empty_list(value): if not value or len(value) < 1: raise ValidationError("Should have more than one element.") @@ -190,6 +238,8 @@ class GitPackageSource(Model): paths = ListType(StringType, serialize_when_none=False) + configs = ListType(StringType, serialize_when_none=False) + class PackageSources(Model): git = ListType(ModelType(GitPackageSource)) diff --git a/stacker/tests/test_config.py b/stacker/tests/test_config.py index 740d07a64..17e5ce53d 100644 --- a/stacker/tests/test_config.py +++ b/stacker/tests/test_config.py @@ -6,7 +6,8 @@ load, render, parse, - dump + dump, + process_remote_sources ) from stacker.config import Config, Stack from stacker.environment import parse_environment @@ -171,7 +172,7 @@ def test_config_build(self): config.validate() def test_parse(self): - config = parse(""" + config_with_lists = """ namespace: prod stacker_bucket: stacker-prod pre_build: @@ -218,52 +219,116 @@ def test_parse(self): requires: ['vpc'] variables: VpcId: ${output vpc::VpcId} - """) + """ + config_with_dicts = """ + namespace: prod + stacker_bucket: stacker-prod + pre_build: + prebuild_createdomain: + path: stacker.hooks.route53.create_domain + required: true + args: + domain: mydomain.com + post_build: + postbuild_createdomain: + path: stacker.hooks.route53.create_domain + required: true + args: + domain: mydomain.com + pre_destroy: + predestroy_createdomain: + path: stacker.hooks.route53.create_domain + required: true + args: + domain: mydomain.com + post_destroy: + postdestroy_createdomain: + path: stacker.hooks.route53.create_domain + required: true + args: + domain: mydomain.com + package_sources: + git: + - uri: git@github.com:acmecorp/stacker_blueprints.git + - uri: git@github.com:remind101/stacker_blueprints.git + tag: 1.0.0 + paths: + - stacker_blueprints + - uri: git@github.com:contoso/webapp.git + branch: staging + - uri: git@github.com:contoso/foo.git + commit: 12345678 + tags: + environment: production + stacks: + vpc: + class_path: blueprints.VPC + variables: + PrivateSubnets: + - 10.0.0.0/24 + bastion: + class_path: blueprints.Bastion + requires: ['vpc'] + variables: + VpcId: ${output vpc::VpcId} + """ - config.validate() + for raw_config in [config_with_lists, config_with_dicts]: + config = parse(raw_config) + + config.validate() + + self.assertEqual(config.namespace, "prod") + self.assertEqual(config.stacker_bucket, "stacker-prod") - self.assertEqual(config.namespace, "prod") - self.assertEqual(config.stacker_bucket, "stacker-prod") + for hooks in [config.pre_build, config.post_build, + config.pre_destroy, config.post_destroy]: + self.assertEqual( + hooks[0].path, "stacker.hooks.route53.create_domain") + self.assertEqual( + hooks[0].required, True) + self.assertEqual( + hooks[0].args, {"domain": "mydomain.com"}) - for hooks in [config.pre_build, config.post_build, - config.pre_destroy, config.post_destroy]: self.assertEqual( - hooks[0].path, "stacker.hooks.route53.create_domain") + config.package_sources.git[0].uri, + "git@github.com:acmecorp/stacker_blueprints.git") + self.assertEqual( + config.package_sources.git[1].uri, + "git@github.com:remind101/stacker_blueprints.git") self.assertEqual( - hooks[0].required, True) + config.package_sources.git[1].tag, + "1.0.0") self.assertEqual( - hooks[0].args, {"domain": "mydomain.com"}) - - self.assertEqual( - config.package_sources.git[0].uri, - "git@github.com:acmecorp/stacker_blueprints.git") - self.assertEqual( - config.package_sources.git[1].uri, - "git@github.com:remind101/stacker_blueprints.git") - self.assertEqual( - config.package_sources.git[1].tag, - "1.0.0") - self.assertEqual( - config.package_sources.git[1].paths, - ["stacker_blueprints"]) - self.assertEqual( - config.package_sources.git[2].branch, - "staging") - - self.assertEqual(config.tags, {"environment": "production"}) - - self.assertEqual(len(config.stacks), 2) - vpc = config.stacks[0] - self.assertEqual(vpc.name, "vpc") - self.assertEqual(vpc.class_path, "blueprints.VPC") - self.assertEqual(vpc.requires, None) - self.assertEqual(vpc.variables, {"PrivateSubnets": ["10.0.0.0/24"]}) - - bastion = config.stacks[1] - self.assertEqual(bastion.name, "bastion") - self.assertEqual(bastion.class_path, "blueprints.Bastion") - self.assertEqual(bastion.requires, ["vpc"]) - self.assertEqual(bastion.variables, {"VpcId": "${output vpc::VpcId}"}) + config.package_sources.git[1].paths, + ["stacker_blueprints"]) + self.assertEqual( + config.package_sources.git[2].branch, + "staging") + + self.assertEqual(config.tags, {"environment": "production"}) + + self.assertEqual(len(config.stacks), 2) + + vpc_index = next( + i for (i, d) in enumerate(config.stacks) if d.name == "vpc" + ) + vpc = config.stacks[vpc_index] + self.assertEqual(vpc.name, "vpc") + self.assertEqual(vpc.class_path, "blueprints.VPC") + self.assertEqual(vpc.requires, None) + self.assertEqual(vpc.variables, + {"PrivateSubnets": ["10.0.0.0/24"]}) + + bastion_index = next( + i for (i, d) in enumerate(config.stacks) if d.name == "bastion" + ) + bastion = config.stacks[bastion_index] + self.assertEqual(bastion.name, "bastion") + self.assertEqual(bastion.class_path, "blueprints.Bastion") + self.assertEqual(bastion.requires, ["vpc"]) + self.assertEqual(bastion.variables, + {"VpcId": "${output vpc::VpcId}"}) def test_dump_complex(self): config = Config({ @@ -305,6 +370,15 @@ def test_load_adds_sys_path(self): load(config) self.assertIn("/foo/bar", sys.path) + def test_process_empty_remote_sources(self): + config = """ + namespace: prod + stacks: + - name: vpc + class_path: blueprints.VPC + """ + self.assertEqual(config, process_remote_sources(config)) + def test_lookup_with_sys_path(self): config = Config({ "sys_path": "stacker/tests", diff --git a/stacker/tests/test_util.py b/stacker/tests/test_util.py index 0b4355416..379e7940c 100644 --- a/stacker/tests/test_util.py +++ b/stacker/tests/test_util.py @@ -14,6 +14,8 @@ load_object_from_string, camel_to_snake, handle_hooks, + merge_map, + yaml_to_ordered_dict, retry_with_backoff, get_client_region, get_s3_endpoint, @@ -66,6 +68,50 @@ def test_camel_to_snake(self): for t in tests: self.assertEqual(camel_to_snake(t[0]), t[1]) + def test_merge_map(self): + tests = [ + # 2 lists of stacks defined + [{'stacks': [{'stack1': {'variables': {'a': 'b'}}}]}, + {'stacks': [{'stack2': {'variables': {'c': 'd'}}}]}, + {'stacks': [ + {'stack1': { + 'variables': { + 'a': 'b'}}}, + {'stack2': { + 'variables': { + 'c': 'd'}}}]}], + # A list of stacks combined with a higher precedence dict of stacks + [{'stacks': [{'stack1': {'variables': {'a': 'b'}}}]}, + {'stacks': {'stack2': {'variables': {'c': 'd'}}}}, + {'stacks': {'stack2': {'variables': {'c': 'd'}}}}], + # 2 dicts of stacks with non-overlapping variables merged + [{'stacks': {'stack1': {'variables': {'a': 'b'}}}}, + {'stacks': {'stack1': {'variables': {'c': 'd'}}}}, + {'stacks': { + 'stack1': { + 'variables': { + 'a': 'b', + 'c': 'd'}}}}], + # 2 dicts of stacks with overlapping variables merged + [{'stacks': {'stack1': {'variables': {'a': 'b'}}}}, + {'stacks': {'stack1': {'variables': {'a': 'c'}}}}, + {'stacks': {'stack1': {'variables': {'a': 'c'}}}}], + ] + for t in tests: + self.assertEqual(merge_map(t[0], t[1]), t[2]) + + def test_yaml_to_ordered_dict(self): + raw_config = """ + pre_build: + hook2: + path: foo.bar + hook1: + path: foo1.bar1 + """ + config = yaml_to_ordered_dict(raw_config) + self.assertEqual(config['pre_build'].keys()[0], 'hook2') + self.assertEqual(config['pre_build']['hook2']['path'], 'foo.bar') + def test_get_client_region(self): regions = ["us-east-1", "us-west-1", "eu-west-1", "sa-east-1"] for region in regions: @@ -99,7 +145,7 @@ def test_SourceProcessor_helpers(self): with mock.patch.object(SourceProcessor, 'create_cache_directories', new=mock_create_cache_directories): - sp = SourceProcessor() + sp = SourceProcessor(sources={}) self.assertEqual( sp.sanitize_git_path('git@github.com:foo/bar.git'), @@ -110,16 +156,20 @@ def test_SourceProcessor_helpers(self): 'git_github.com_foo_bar-v1' ) - self.assertEqual( - sp.determine_git_ls_remote_ref( - GitPackageSource({'branch': 'foo'})), - 'refs/heads/foo' - ) + for i in [GitPackageSource({'branch': 'foo'}), {'branch': 'foo'}]: + self.assertEqual( + sp.determine_git_ls_remote_ref(i), + 'refs/heads/foo' + ) for i in [{'uri': 'git@foo'}, {'tag': 'foo'}, {'commit': '1234'}]: self.assertEqual( sp.determine_git_ls_remote_ref(GitPackageSource(i)), 'HEAD' ) + self.assertEqual( + sp.determine_git_ls_remote_ref(i), + 'HEAD' + ) self.assertEqual( sp.git_ls_remote('https://github.com/remind101/stacker.git', @@ -138,6 +188,8 @@ def test_SourceProcessor_helpers(self): for i in bad_configs: with self.assertRaises(ImportError): sp.determine_git_ref(GitPackageSource(i)) + with self.assertRaises(ImportError): + sp.determine_git_ref(i) self.assertEqual( sp.determine_git_ref( @@ -151,11 +203,19 @@ def test_SourceProcessor_helpers(self): GitPackageSource({'uri': 'git@foo', 'commit': '1234'})), '1234' ) + self.assertEqual( + sp.determine_git_ref({'uri': 'git@foo', 'commit': '1234'}), + '1234' + ) self.assertEqual( sp.determine_git_ref( GitPackageSource({'uri': 'git@foo', 'tag': 'v1.0.0'})), 'v1.0.0' ) + self.assertEqual( + sp.determine_git_ref({'uri': 'git@foo', 'tag': 'v1.0.0'}), + 'v1.0.0' + ) hook_queue = Queue.Queue() diff --git a/stacker/util.py b/stacker/util.py index 41ac049a5..2001cabe7 100644 --- a/stacker/util.py +++ b/stacker/util.py @@ -10,6 +10,8 @@ import sys import tempfile import time +import yaml +from collections import OrderedDict from git import Repo import botocore.exceptions @@ -265,6 +267,49 @@ def load_object_from_string(fqcn): return getattr(sys.modules[module_path], object_name) +def merge_map(a, b): + """Recursively merge elements of argument b into argument a. + + Primarly used for merging two dictionaries together, where dict b takes + precedence over dict a. If 2 lists are provided, they are concatenated. + """ + if isinstance(a, list) and isinstance(b, list): + return a + b + + if not isinstance(a, dict) or not isinstance(b, dict): + return b + + for key in b.keys(): + a[key] = merge_map(a[key], b[key]) if key in a else b[key] + return a + + +def yaml_to_ordered_dict(stream, loader=yaml.SafeLoader): + """Provides yaml.load alternative with preserved dictionary order. + + Args: + stream (string): YAML string to load. + loader (:class:`yaml.loader`): PyYAML loader class. Defaults to safe + load. + + Returns: + OrderedDict: Parsed YAML. + """ + class OrderedLoader(loader): + """Subclass of Python yaml class.""" + pass + + def construct_mapping(loader, node): + """Override parent method to use OrderedDict.""" + loader.flatten_mapping(node) + return OrderedDict(loader.construct_pairs(node)) + + OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping) + return yaml.load(stream, OrderedLoader) + + def uppercase_first_letter(s): """Return string "s" with first character upper case.""" return s[0].upper() + s[1:] @@ -459,19 +504,22 @@ class SourceProcessor(): """Makes remote python package sources available in the running python environment.""" - def __init__(self, stacker_cache_dir=None): + def __init__(self, sources, stacker_cache_dir=None): """ - Processes a config's list of package sources + Processes a config's defined package sources. Args: - stacker_cache_dir (string): Directory of stacker local cache. - Default to $HOME/.stacker + sources (dict): Package sources from Stacker config dictionary + stacker_cache_dir (string): Path where remote sources will be + cached. """ - stacker_cache_dir = (stacker_cache_dir or - os.path.expanduser("~/.stacker")) + if not stacker_cache_dir: + stacker_cache_dir = os.path.expanduser("~/.stacker") package_cache_dir = os.path.join(stacker_cache_dir, 'packages') self.stacker_cache_dir = stacker_cache_dir self.package_cache_dir = package_cache_dir + self.sources = sources + self.configs_to_merge = [] self.create_cache_directories() def create_cache_directories(self): @@ -481,30 +529,11 @@ def create_cache_directories(self): os.mkdir(self.stacker_cache_dir) os.mkdir(self.package_cache_dir) - def get_package_sources(self, sources): - """Makes remote python packages available for local use - - Example:: - - {'git': [ - {'uri': 'git@github.com:remind101/stacker_blueprints.git', - 'tag': '1.0.0', - 'paths': ['stacker_blueprints']}, - {'uri': 'git@github.com:acmecorp/stacker_blueprints.git'}, - {'uri': 'git@github.com:contoso/webapp.git', - 'branch': 'staging'}, - {'uri': 'git@github.com:contoso/foo.git', - 'commit': '12345678'} - ]} - - Args: - sources (dict): Dictionary of remote sources from config. - Currently supports git repositories - """ + def get_package_sources(self): + """Makes remote python packages available for local use.""" # Checkout git repositories specified in config - if sources.git: - for config in sources.git: - self.fetch_git_package(config=config) + for config in self.sources.get('git', []): + self.fetch_git_package(config=config) def fetch_git_package(self, config): """Makes a remote git repository available for local use @@ -515,7 +544,7 @@ def fetch_git_package(self, config): """ ref = self.determine_git_ref(config) - dir_name = self.sanitize_git_path(uri=config.uri, ref=ref) + dir_name = self.sanitize_git_path(uri=config['uri'], ref=ref) cached_dir_path = os.path.join(self.package_cache_dir, dir_name) # We can skip cloning the repo if it's already been cached @@ -523,7 +552,7 @@ def fetch_git_package(self, config): tmp_dir = tempfile.mkdtemp(prefix='stacker') try: tmp_repo_path = os.path.join(tmp_dir, dir_name) - with Repo.clone_from(config.uri, tmp_repo_path) as repo: + with Repo.clone_from(config['uri'], tmp_repo_path) as repo: repo.head.reference = ref repo.head.reset(index=True, working_tree=True) shutil.move(tmp_repo_path, self.package_cache_dir) @@ -532,19 +561,24 @@ def fetch_git_package(self, config): # Cloning (if necessary) is complete. Now add the appropriate # directory (or directories) to sys.path - if config.paths: - for path in config.paths: + if config.get('paths'): + for path in config['paths']: path_to_append = os.path.join(self.package_cache_dir, dir_name, path) logger.debug("Appending \"%s\" to python sys.path", path_to_append) - sys.path.append(os.path.join(self.package_cache_dir, - dir_name, - path)) + sys.path.append(path_to_append) else: sys.path.append(cached_dir_path) + # If the configuration defines a set of remote config yamls to + # include, add them to the list for merging + if config.get('configs'): + for config_filename in config['configs']: + self.configs_to_merge.append(os.path.join(cached_dir_path, + config_filename)) + def git_ls_remote(self, uri, ref): """Determines the latest commit id for a given ref. @@ -578,8 +612,8 @@ def determine_git_ls_remote_ref(self, config): Returns: str: A branch reference or "HEAD" """ - if config.branch: - ref = "refs/heads/%s" % config.branch + if config.get('branch'): + ref = "refs/heads/%s" % config['branch'] else: ref = "HEAD" @@ -609,15 +643,15 @@ def determine_git_ref(self, config): # Now check for a specific point in time referenced and return it if # present - if config.commit: - ref = config.commit - elif config.tag: - ref = config.tag + if config.get('commit'): + ref = config['commit'] + elif config.get('tag'): + ref = config['tag'] else: # Since a specific commit/tag point in time has not been specified, # check the remote repo for the commit id to use ref = self.git_ls_remote( - config.uri, + config['uri'], self.determine_git_ls_remote_ref(config) ) return ref