Merge pull request #26 from remind101/remove_dns_dependency_add_plugins

Remove dns requirement, add hook/plugin support

README.rst

@@ -33,18 +33,24 @@ if you'd like to play with something smaller. To launch the stacks, after
 installing stacker and loading your AWS API keys in your environment
 (AWS\_ACCESS\_KEY\_ID & AWS\_SECRET\_ACCESS\_KEY), call the following::
 
-    stacker -v -r us-east-1 -d example.com -p CidrBlock=10.128.0.0/16 conf/example.yaml
+    stacker -v -p BaseDomain=example.com -r us-east-1 -p AZCount=2 -p CidrBlock=10.128.0.0/16 example.com conf/example.yaml
 
 Here's the syntax help from the command::
 
   # stacker -h
-  usage: stacker [-h] [-r REGION] [-m MAX_ZONES] [-v] [-d DOMAIN]
-                 [-p PARAMETER=VALUE] [--stacks STACKNAME]
-                 config
+  usage: stacker [-h] [-r REGION] [-m MAX_ZONES] [-v] [-p PARAMETER=VALUE]
+                 [--stacks STACKNAME]
+                 namespace config
 
-  Launches AWS Cloudformation stacks from config.
+  Launches or updates cloudformation stacks based on the given config. The
+  script is smart enough to figure out if anything (the template, or parameters)
+  has changed for a given stack. If not, it will skip that stack. Can also pull
+  parameters from other stack's outputs.
 
   positional arguments:
+    namespace             The namespace for the stack collection. This will be
+                          used as the prefix to the cloudformation stacks as
+                          well as the s3 bucket where templates are stored.
     config                The config file where stack configuration is located.
                           Must be in yaml format.
 
@@ -59,13 +65,10 @@ Here's the syntax help from the command::
                           availability zones.
     -v, --verbose         Increase output verbosity. May be specified up to
                           twice.
-    -d DOMAIN, --domain DOMAIN
-                          The domain to run in. Gets converted into the
-                          BaseDomain Parameter for use in stack templates.
     -p PARAMETER=VALUE, --parameter PARAMETER=VALUE
-                          Adds parameters from the command line that can be
-                          used inside any of the stacks being built. Can be
-                          specified more than once.
+                          Adds parameters from the command line that can be used
+                          inside any of the stacks being built. Can be specified
+                          more than once.
     --stacks STACKNAME    Only work on the stacks given. Can be specified more
                           than once. If not specified then stacker will work on
                           all stacks in the config file.

conf/example.yaml

@@ -1,3 +1,14 @@
+# Hooks require a path.
+# If the build should stop when a hook fails, set required to true.
+# pre_build happens before the build
+# post_build happens after the build
+pre_build:
+  - path: stacker.hooks.route53.create_domain
+    required: true
+    # Additional args can be passed as a dict of key/value pairs in kwargs
+    # kwargs:
+# post_build:
+
 mappings:
   AmiMap:
     us-east-1:

scripts/stacker

@@ -13,7 +13,9 @@ from collections import Mapping
 import copy
 
 import yaml
+
 from stacker.builder import Builder
+from stacker.util import handle_hooks
 
 logger = logging.getLogger()
 
@@ -73,10 +75,6 @@ def parse_args():
     parser.add_argument('-v', '--verbose', action='count', default=0,
                         help='Increase output verbosity. May be specified up '
                              'to twice.')
-    parser.add_argument('-d', '--domain',
-                        help="The domain to run in. Gets converted into the "
-                             "BaseDomain Parameter for use in stack "
-                             "templates.")
     parser.add_argument("-p", "--parameter", dest="parameters",
                         metavar="PARAMETER=VALUE", type=key_value_arg,
                         action=KeyValueAction, default={},
@@ -89,10 +87,14 @@ def parse_args():
                              "specified more than once. If not specified "
                              "then stacker will work on all stacks in the "
                              "config file.")
+    parser.add_argument('namespace',
+                        help='The namespace for the stack collection. This '
+                             'will be used as the prefix to the '
+                             'cloudformation stacks as well as the s3 bucket '
+                             'where templates are stored.')
     parser.add_argument('config',
                         help="The config file where stack configuration is "
                              "located. Must be in yaml format.")
-
     return parser.parse_args()
 
 
@@ -112,15 +114,20 @@ if __name__ == '__main__':
     args = parse_args()
     setup_logging(args.verbose)
     parameters = copy.deepcopy(args.parameters)
-    parameters['BaseDomain'] = args.domain
 
     with open(args.config) as fd:
         config = yaml.load(fd)
 
-    builder = Builder(args.region, mappings=config['mappings'],
+    mappings = config['mappings']
+
+    builder = Builder(args.region, args.namespace, mappings=mappings,
                       parameters=parameters)
 
     stack_definitions = get_stack_definitions(config, args.stacks)
     stack_names = [s['name'] for s in stack_definitions]
+    handle_hooks('pre_build', config.get('pre_build', None), args.region,
+                 args.namespace, mappings, parameters)
     logger.info("Working on stacks: %s", ', '.join(stack_names))
     builder.build(stack_definitions)
+    handle_hooks('post_build', config.get('post_build', None), args.region,
+                 args.namespace, mappings, parameters)

stacker/blueprints/asg.py

@@ -22,6 +22,7 @@ class AutoscalingGroup(Blueprint):
                       'description': 'Top level security group.'},
         'BaseDomain': {
             'type': 'String',
+            'default': '',
             'description': 'Base domain for the stack.'},
         'PrivateSubnets': {'type': 'List<AWS::EC2::Subnet::Id>',
                            'description': 'Subnets to deploy private '
@@ -61,12 +62,18 @@ def create_conditions(self):
         self.template.add_condition(
             "CreateELB",
             Not(Equals(Ref("ELBHostName"), "")))
+        self.template.add_condition(
+            "SetupDNS",
+            Not(Equals(Ref("BaseDomain"), "")))
         self.template.add_condition(
             "UseSSL",
             Not(Equals(Ref("ELBCertName"), "")))
         self.template.add_condition(
             "CreateSSLELB",
             And(Condition("CreateELB"), Condition("UseSSL")))
+        self.template.add_condition(
+            "SetupELBDNS",
+            And(Condition("CreateELB"), Condition("SetupDNS")))
 
     def create_security_groups(self):
         t = self.template
@@ -156,7 +163,7 @@ def create_load_balancer(self):
                 TTL='120',
                 ResourceRecords=[
                     GetAtt(elb_name, 'DNSName')],
-                Condition="CreateELB"))
+                Condition="SetupELBDNS"))
 
     def create_autoscaling_group(self):
         name = "%sASG" % self.name

stacker/blueprints/base.py

@@ -14,7 +14,7 @@ class Blueprint(object):
         will be created from the class name automatically.
 
     :type context: BlueprintContext object
-    :param config: Used for configuring the Blueprint.
+    :param context: Used for configuring the Blueprint.
 
     :type mappings: dict
     :param mappings: Cloudformation Mappings to be used in the template.

stacker/blueprints/vpc.py

@@ -21,6 +21,7 @@ class VPC(Blueprint):
     PARAMETERS = {
         "AZCount": {
             "type": "Number",
+            "default": "2",
             "description": "The number of AZs to build the VPC in. NOTE: "
                            "this is used by stacker, not by cloudformation "
                            "directly."},

stacker/builder.py

@@ -7,8 +7,7 @@
 
 from boto.exception import S3ResponseError, BotoServerError
 
-from .util import (create_route53_zone, load_object_from_string,
-                   get_bucket_location)
+from .util import load_object_from_string, get_bucket_location
 
 from .plan import (Plan, INPROGRESS_STATUSES, STATUS_SUBMITTED,
                    COMPLETE_STATUSES)
@@ -25,17 +24,17 @@ def __str__(self):
         return self.message
 
 
-def get_stack_full_name(cfn_domain, stack_name):
-    return "%s-%s" % (cfn_domain, stack_name)
+def get_stack_full_name(cfn_base, stack_name):
+    return "%s-%s" % (cfn_base, stack_name)
 
 
 def stack_template_key_name(blueprint):
     return "%s-%s.json" % (blueprint.name, blueprint.version)
 
 
-def stack_template_url(cfn_domain, blueprint):
+def stack_template_url(bucket_name, blueprint):
     key_name = stack_template_key_name(blueprint)
-    return "https://s3.amazonaws.com/%s/%s" % (cfn_domain, key_name)
+    return "https://s3.amazonaws.com/%s/%s" % (bucket_name, key_name)
 
 
 class Builder(object):
@@ -56,12 +55,13 @@ class Builder(object):
     allowing you to pull information from one stack and use it in another.
     """
 
-    def __init__(self, region, mappings=None, parameters=None):
+    def __init__(self, region, namespace, mappings=None, parameters=None):
         self.region = region
-        self.domain = parameters["BaseDomain"]
         self.mappings = mappings or {}
         self.parameters = parameters or {}
-        self.cfn_domain = self.domain.replace('.', '-')
+        self.namespace = namespace
+        self.cfn_base = namespace.replace('.', '-').lower()
+        self.bucket_name = "stacker-%s" % self.cfn_base
 
         self._conn = None
         self._cfn_bucket = None
@@ -79,16 +79,20 @@ def cfn_bucket(self):
         if not getattr(self, '_cfn_bucket', None):
             s3 = self.conn.s3
             try:
-                self._cfn_bucket = s3.get_bucket(self.cfn_domain)
+                self._cfn_bucket = s3.get_bucket(self.bucket_name)
             except S3ResponseError, e:
                 if e.error_code == 'NoSuchBucket':
-                    logger.debug("Creating bucket %s.", self.cfn_domain)
+                    logger.debug("Creating bucket %s.", self.bucket_name)
                     self._cfn_bucket = s3.create_bucket(
-                        self.cfn_domain,
+                        self.bucket_name,
                         location=get_bucket_location(self.region))
+                elif e.error_code == 'AccessDenied':
+                    logger.exception("Access denied for bucket %s.",
+                                     self.bucket_name)
+                    raise
                 else:
                     logger.exception("Error creating bucket %s.",
-                                     self.cfn_domain)
+                                     self.bucket_name)
                     raise
         return self._cfn_bucket
 
@@ -97,10 +101,10 @@ def reset(self):
         self.outputs = {}
 
     def get_stack_full_name(self, stack_name):
-        return get_stack_full_name(self.cfn_domain, stack_name)
+        return get_stack_full_name(self.cfn_base, stack_name)
 
     def stack_template_url(self, blueprint):
-        return stack_template_url(self.cfn_domain, blueprint)
+        return stack_template_url(self.bucket_name, blueprint)
 
     def s3_stack_push(self, blueprint, force=False):
         """ Pushes the rendered blueprint's template to S3.
@@ -118,11 +122,10 @@ def s3_stack_push(self, blueprint, force=False):
             return template_url
         key = self.cfn_bucket.new_key(key_name)
         key.set_contents_from_string(blueprint.rendered)
+        logger.debug("Blueprint %s pushed to %s.", blueprint.name,
+                     template_url)
         return template_url
 
-    def setup_prereqs(self):
-        create_route53_zone(self.conn.route53, self.domain)
-
     def get_stack_status(self, stack_name):
         """ Get the status of a CloudFormation stack. """
         stack_info = self.conn.cloudformation.describe_stacks(stack_name)
@@ -262,6 +265,7 @@ def launch_stack(self, stack_name, blueprint):
             logger.debug("Stack %s in progress with %s status.",
                          full_name, stack.stack_status)
             return
+        logger.info("Launching stack %s now.", stack_name)
         template_url = self.s3_stack_push(blueprint)
         tags = self.build_stack_tags(stack_context, template_url)
         parameters = self.resolve_parameters(stack_context.parameters,
@@ -330,6 +334,7 @@ def build_plan(self, stack_definitions):
         for stack_def in stack_definitions:
             # Combine the Builder parameters with the stack parameters
             stack_def['parameters'].update(self.parameters)
+            stack_def['namespace'] = self.namespace
             plan.add(stack_def)
         return plan
 
@@ -339,7 +344,6 @@ def build(self, stack_definitions):
         This is the main entry point for the Builder.
         """
         self.reset()
-        self.setup_prereqs()
         self.plan = self.build_plan(stack_definitions)
         logger.info("Launching stacks: %s", ', '.join(self.plan.keys()))
 
@@ -360,8 +364,6 @@ def build(self, stack_definitions):
                     logger.debug("Stack %s waiting on required stacks: "
                                  "%s", stack_name, ', '.join(pending_required))
                     continue
-                logger.debug("All required stacks are finished, building %s "
-                             "now.", stack_name)
                 blueprint = self.build_blueprint(stack_context)
                 self.launch_stack(stack_name, blueprint)
             time.sleep(5)

stacker/hooks/route53.py

@@ -0,0 +1,19 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from aws_helper.connection import ConnectionManager
+
+from stacker.util import create_route53_zone
+
+
+def create_domain(region, namespace, mappings, parameters, **kwargs):
+    conn = ConnectionManager(region)
+    try:
+        domain = parameters['BaseDomain']
+    except KeyError:
+        logger.error("BaseDomain parameter not provided.")
+        return False
+    create_route53_zone(conn.route53, domain)
+    return True

stacker/plan.py

@@ -15,9 +15,11 @@
 
 
 class BlueprintContext(object):
-    def __init__(self, name, class_path, requires=None, parameters=None):
+    def __init__(self, name, class_path, namespace, requires=None,
+                 parameters=None):
         self.name = name
         self.class_path = class_path
+        self.namespace = namespace
         self.parameters = parameters or {}
         requires = requires or []
         self._requires = set(requires)

stacker/tests/test_plan.py

@@ -8,6 +8,7 @@ def generate_definition(base_name, _id):
         "name": "%s.%d" % (base_name, _id),
         "class_path": "stacker.blueprints.%s.%s" % (base_name,
                                                     base_name.upper()),
+        "namespace": "example-com",
         "parameters": {
             "ExternalParameter": "fakeStack2::FakeParameter",
             "InstanceType": "m3.medium",