Skip to content

Commit

Permalink
add remote config support (#458)
Browse files Browse the repository at this point in the history 
* add remote config support

Extends package_sources to support remote configuration files.

* update docs about hook dictionaries

* cleanup for code review

* additional cleanups

* Clean up the dict -> list stuff
  • Loading branch information
@troyready @phobologic
troyready authored and phobologic committed Oct 2, 2017
1 parent 6c31a11 commit f307e5d
Show file tree
Hide file tree
Showing 5 changed files with 354 additions and 101 deletions.
37 changes: 36 additions & 1 deletion docs/config.rst
View file
Expand Up @@ -146,6 +146,42 @@ Cloned repositories will be cached between builds; the cache location defaults
to ~/.stacker but can be manually specified via the **stacker_cache_dir** top
level keyword.

Remote Configs
~~~~~~~~~~~~~~
Configuration yamls from remote configs can also be used by specifying a list
of ``configs`` in the repo to use::

package_sources:
git:
- uri: git@github.com:acmecorp/stacker_blueprints.git
configs:
- vpc.yaml

In this example, the configuration in ``vpc.yaml`` will be merged into the
running current configuration, with the current configuration's values taking
priority over the values in ``vpc.yaml``.

Dictionary Stack Names & Hook Paths
:::::::::::::::::::::::::::::::::::
To allow remote configs to be selectively overriden, stack names & hook
paths can optionally be defined as dictionaries, e.g.::

pre_build:
my_route53_hook:
path: stacker.hooks.route53.create_domain:
required: true
args:
domain: mydomain.com
stacks:
vpc-example:
class_path: stacker_blueprints.vpc.VPC
locked: false
enabled: true
bastion-example:
class_path: stacker_blueprints.bastion.Bastion
locked: false
enabled: true

Pre & Post Hooks
----------------

Expand Down Expand Up @@ -316,7 +352,6 @@ Here's an example from stacker_blueprints_, used to create a VPC::
- 10.128.20.0/22
CidrBlock: 10.128.0.0/16


Variables
==========

Expand Down
68 changes: 59 additions & 9 deletions stacker/config/__init__.py
View file
@@ -1,3 +1,4 @@
import copy
import sys
import logging

Expand All @@ -19,7 +20,7 @@
import yaml

from ..lookups import register_lookup_handler
from ..util import SourceProcessor
from ..util import merge_map, yaml_to_ordered_dict, SourceProcessor
from .. import exceptions

# register translators (yaml constructors)
Expand All @@ -43,7 +44,9 @@ def render_parse_load(raw_config, environment=None, validate=True):
"""

rendered = render(raw_config, environment)
pre_rendered = render(raw_config, environment)

rendered = process_remote_sources(pre_rendered, environment)

config = parse(rendered)

Expand Down Expand Up @@ -106,6 +109,23 @@ def parse(raw_config):
"""

# Convert any applicable dictionaries back into lists
# This is necessary due to the move from lists for these top level config
# values to either lists or OrderedDicts.
# Eventually we should probably just make them OrderedDicts only.
config_dict = yaml_to_ordered_dict(raw_config)
for top_level_key in ['stacks', 'pre_build', 'post_build', 'pre_destroy',
'post_destroy']:
top_level_value = config_dict.get(top_level_key)
if isinstance(top_level_value, dict):
tmp_list = []
for key, value in top_level_value.iteritems():
tmp_dict = copy.deepcopy(value)
if top_level_key == 'stacks':
tmp_dict['name'] = key
tmp_list.append(tmp_dict)
config_dict[top_level_key] = tmp_list

# We have to enable non-strict mode, because people may be including top
# level keys for re-use with stacks (e.g. including something like
# `common_variables: &common_variables`).
Expand All @@ -118,7 +138,7 @@ def parse(raw_config):
# should consider enabling this in the future.
strict = False

return Config(yaml.safe_load(raw_config), strict=strict)
return Config(config_dict, strict=strict)


def load(config):
Expand All @@ -140,12 +160,6 @@ def load(config):
if config.lookups:
for key, handler in config.lookups.iteritems():
register_lookup_handler(key, handler)
sources = config.package_sources
if sources is not None:
processor = SourceProcessor(
stacker_cache_dir=config.stacker_cache_dir
)
processor.get_package_sources(sources=sources)

return config

Expand All @@ -169,6 +183,40 @@ def dump(config):
allow_unicode=True)


def process_remote_sources(raw_config, environment=None):
"""Stage remote package sources and merge in remote configs.
Args:
raw_config (str): the raw stacker configuration string.
environment (dict, optional): any environment values that should be
passed to the config
Returns:
str: the raw stacker configuration string
"""

config = yaml.safe_load(raw_config)
if config.get('package_sources'):
processor = SourceProcessor(
sources=config['package_sources'],
stacker_cache_dir=config.get('stacker_cache_dir')
)
processor.get_package_sources()
if processor.configs_to_merge:
for i in processor.configs_to_merge:
logger.debug("Merging in remote config \"%s\"", i)
remote_config = yaml.safe_load(open(i))
config = merge_map(remote_config, config)
# Call the render again as the package_sources may have merged in
# additional environment lookups
if not environment:
environment = {}
return render(str(config), environment)

return raw_config


def not_empty_list(value):
if not value or len(value) < 1:
raise ValidationError("Should have more than one element.")
Expand All @@ -190,6 +238,8 @@ class GitPackageSource(Model):

paths = ListType(StringType, serialize_when_none=False)

configs = ListType(StringType, serialize_when_none=False)


class PackageSources(Model):
git = ListType(ModelType(GitPackageSource))
Expand Down
158 changes: 116 additions & 42 deletions stacker/tests/test_config.py
View file
Expand Up @@ -6,7 +6,8 @@
load,
render,
parse,
dump
dump,
process_remote_sources
)
from stacker.config import Config, Stack
from stacker.environment import parse_environment
Expand Down Expand Up @@ -171,7 +172,7 @@ def test_config_build(self):
config.validate()

def test_parse(self):
config = parse("""
config_with_lists = """
namespace: prod
stacker_bucket: stacker-prod
pre_build:
Expand Down Expand Up @@ -218,52 +219,116 @@ def test_parse(self):
requires: ['vpc']
variables:
VpcId: ${output vpc::VpcId}
""")
"""
config_with_dicts = """
namespace: prod
stacker_bucket: stacker-prod
pre_build:
prebuild_createdomain:
path: stacker.hooks.route53.create_domain
required: true
args:
domain: mydomain.com
post_build:
postbuild_createdomain:
path: stacker.hooks.route53.create_domain
required: true
args:
domain: mydomain.com
pre_destroy:
predestroy_createdomain:
path: stacker.hooks.route53.create_domain
required: true
args:
domain: mydomain.com
post_destroy:
postdestroy_createdomain:
path: stacker.hooks.route53.create_domain
required: true
args:
domain: mydomain.com
package_sources:
git:
- uri: git@github.com:acmecorp/stacker_blueprints.git
- uri: git@github.com:remind101/stacker_blueprints.git
tag: 1.0.0
paths:
- stacker_blueprints
- uri: git@github.com:contoso/webapp.git
branch: staging
- uri: git@github.com:contoso/foo.git
commit: 12345678
tags:
environment: production
stacks:
vpc:
class_path: blueprints.VPC
variables:
PrivateSubnets:
- 10.0.0.0/24
bastion:
class_path: blueprints.Bastion
requires: ['vpc']
variables:
VpcId: ${output vpc::VpcId}
"""

config.validate()
for raw_config in [config_with_lists, config_with_dicts]:
config = parse(raw_config)

config.validate()

self.assertEqual(config.namespace, "prod")
self.assertEqual(config.stacker_bucket, "stacker-prod")

self.assertEqual(config.namespace, "prod")
self.assertEqual(config.stacker_bucket, "stacker-prod")
for hooks in [config.pre_build, config.post_build,
config.pre_destroy, config.post_destroy]:
self.assertEqual(
hooks[0].path, "stacker.hooks.route53.create_domain")
self.assertEqual(
hooks[0].required, True)
self.assertEqual(
hooks[0].args, {"domain": "mydomain.com"})

for hooks in [config.pre_build, config.post_build,
config.pre_destroy, config.post_destroy]:
self.assertEqual(
hooks[0].path, "stacker.hooks.route53.create_domain")
config.package_sources.git[0].uri,
"git@github.com:acmecorp/stacker_blueprints.git")
self.assertEqual(
config.package_sources.git[1].uri,
"git@github.com:remind101/stacker_blueprints.git")
self.assertEqual(
hooks[0].required, True)
config.package_sources.git[1].tag,
"1.0.0")
self.assertEqual(
hooks[0].args, {"domain": "mydomain.com"})

self.assertEqual(
config.package_sources.git[0].uri,
"git@github.com:acmecorp/stacker_blueprints.git")
self.assertEqual(
config.package_sources.git[1].uri,
"git@github.com:remind101/stacker_blueprints.git")
self.assertEqual(
config.package_sources.git[1].tag,
"1.0.0")
self.assertEqual(
config.package_sources.git[1].paths,
["stacker_blueprints"])
self.assertEqual(
config.package_sources.git[2].branch,
"staging")

self.assertEqual(config.tags, {"environment": "production"})

self.assertEqual(len(config.stacks), 2)
vpc = config.stacks[0]
self.assertEqual(vpc.name, "vpc")
self.assertEqual(vpc.class_path, "blueprints.VPC")
self.assertEqual(vpc.requires, None)
self.assertEqual(vpc.variables, {"PrivateSubnets": ["10.0.0.0/24"]})

bastion = config.stacks[1]
self.assertEqual(bastion.name, "bastion")
self.assertEqual(bastion.class_path, "blueprints.Bastion")
self.assertEqual(bastion.requires, ["vpc"])
self.assertEqual(bastion.variables, {"VpcId": "${output vpc::VpcId}"})
config.package_sources.git[1].paths,
["stacker_blueprints"])
self.assertEqual(
config.package_sources.git[2].branch,
"staging")

self.assertEqual(config.tags, {"environment": "production"})

self.assertEqual(len(config.stacks), 2)

vpc_index = next(
i for (i, d) in enumerate(config.stacks) if d.name == "vpc"
)
vpc = config.stacks[vpc_index]
self.assertEqual(vpc.name, "vpc")
self.assertEqual(vpc.class_path, "blueprints.VPC")
self.assertEqual(vpc.requires, None)
self.assertEqual(vpc.variables,
{"PrivateSubnets": ["10.0.0.0/24"]})

bastion_index = next(
i for (i, d) in enumerate(config.stacks) if d.name == "bastion"
)
bastion = config.stacks[bastion_index]
self.assertEqual(bastion.name, "bastion")
self.assertEqual(bastion.class_path, "blueprints.Bastion")
self.assertEqual(bastion.requires, ["vpc"])
self.assertEqual(bastion.variables,
{"VpcId": "${output vpc::VpcId}"})

def test_dump_complex(self):
config = Config({
Expand Down Expand Up @@ -305,6 +370,15 @@ def test_load_adds_sys_path(self):
load(config)
self.assertIn("/foo/bar", sys.path)

def test_process_empty_remote_sources(self):
config = """
namespace: prod
stacks:
- name: vpc
class_path: blueprints.VPC
"""
self.assertEqual(config, process_remote_sources(config))

def test_lookup_with_sys_path(self):
config = Config({
"sys_path": "stacker/tests",
Expand Down

0 comments on commit f307e5d

Please sign in to comment.