Merge pull request #34 from mhahn/master

Random fixes and cleanup

scripts/stacker

@@ -8,10 +8,9 @@ Can also pull parameters from other stack's outputs.
 """
 
 import argparse
-import logging
 from collections import Mapping
 import copy
-
+import logging
 import yaml
 
 from stacker.builder import Builder

setup.py

@@ -13,6 +13,7 @@
 
 tests_require = [
     'nose>=1.0',
+    'mock>=1.0',
 ]
 
 

stacker/blueprints/base.py

@@ -1,9 +1,9 @@
-import logging
 import hashlib
+import logging
 
-logger = logging.getLogger(__name__)
+from troposphere import Parameter, Template
 
-from troposphere import Template, Parameter
+logger = logging.getLogger(__name__)
 
 
 class Blueprint(object):

stacker/blueprints/vpc.py

@@ -4,7 +4,7 @@
 """
 
 from troposphere import (
-    Ref, Output, Join, FindInMap, Select, GetAZs, Not, Equals, Tags
+    Ref, Output, Join, FindInMap, Select, GetAZs, Not, Equals, Tags, And, Or
 )
 from troposphere import ec2
 from troposphere.route53 import HostedZone, HostedZoneVPCs
@@ -65,8 +65,17 @@ class VPC(Blueprint):
 
     def create_conditions(self):
         self.template.add_condition(
-            "CreateInternalDomain",
-            Not(Equals(Ref("InternalDomain"), "")))
+            "NoHostedZones",
+            And(
+                Equals(Ref("InternalDomain"), ""),
+                Equals(Ref("BaseDomain"), ""),
+            ))
+        self.template.add_condition(
+            "HasHostedZones",
+            Or(
+                Not(Equals(Ref("InternalDomain"), "")),
+                Not(Equals(Ref("BaseDomain"), "")),
+            ))
 
     def create_vpc(self):
         t = self.template
@@ -82,16 +91,22 @@ def create_internal_zone(self):
         t = self.template
         t.add_resource(
             HostedZone(
-                "EmpireInternalZone",
+                "InternalZone",
                 Name=Ref("InternalDomain"),
                 VPCs=[HostedZoneVPCs(
                     VPCId=VPC_ID,
                     VPCRegion=Ref("AWS::Region"))],
-                Condition="CreateInternalDomain"))
-        t.add_output(Output("InternalZoneId",
-                     Value=Ref("EmpireInternalZone")))
-        t.add_output(Output("InternalZoneName",
-                     Value=Ref("InternalDomain")))
+                Condition="HasHostedZones"))
+        t.add_output(
+            Output(
+                "InternalZoneId",
+                Value=Ref("InternalZone"),
+                Condition="HasHostedZones"))
+        t.add_output(
+            Output(
+                "InternalZoneName",
+                Value=Ref("InternalDomain"),
+                Condition="HasHostedZones"))
 
     def create_default_security_group(self):
         t = self.template
@@ -103,17 +118,35 @@ def create_default_security_group(self):
             Output('DefaultSG',
                    Value=Ref(DEFAULT_SG)))
 
-    def create_dhcp_options(self):
+    def _dhcp_options_hosted_zones(self):
         t = self.template
         domain_name = Join(" ", [Ref("BaseDomain"), Ref("InternalDomain")])
         dhcp_options = t.add_resource(ec2.DHCPOptions(
-            'DHCPOptions',
+            'DHCPOptionsWithDNS',
             DomainName=domain_name,
-            DomainNameServers=['AmazonProvidedDNS', ]))
+            DomainNameServers=['AmazonProvidedDNS', ],
+            Condition="HasHostedZones"))
         t.add_resource(ec2.VPCDHCPOptionsAssociation(
-            'DHCPAssociation',
+            'DHCPAssociationWithDNS',
             VpcId=VPC_ID,
-            DhcpOptionsId=Ref(dhcp_options)))
+            DhcpOptionsId=Ref(dhcp_options),
+            Condition="HasHostedZones"))
+
+    def _dhcp_options_no_hosted_zones(self):
+        t = self.template
+        dhcp_options = t.add_resource(ec2.DHCPOptions(
+            'DHCPOptionsNoDNS',
+            DomainNameServers=['AmazonProvidedDNS', ],
+            Condition="NoHostedZones"))
+        t.add_resource(ec2.VPCDHCPOptionsAssociation(
+            'DHCPAssociationNoDNS',
+            VpcId=VPC_ID,
+            DhcpOptionsId=Ref(dhcp_options),
+            Condition="NoHostedZones"))
+
+    def create_dhcp_options(self):
+        self._dhcp_options_hosted_zones()
+        self._dhcp_options_no_hosted_zones()
 
     def create_gateway(self):
         t = self.template

stacker/builder.py

@@ -1,27 +1,31 @@
+import copy
 import logging
 import time
-import copy
-
-logger = logging.getLogger(__name__)
 
 from aws_helper.connection import ConnectionManager
+from boto.exception import BotoServerError, S3ResponseError
 
-from boto.exception import S3ResponseError, BotoServerError
+from .plan import Plan, INPROGRESS_STATUSES, STATUS_SUBMITTED, COMPLETE_STATUSES
+from .util import get_bucket_location, load_object_from_string
 
-from .util import load_object_from_string, get_bucket_location
-
-from .plan import (Plan, INPROGRESS_STATUSES, STATUS_SUBMITTED,
-                   COMPLETE_STATUSES)
+logger = logging.getLogger(__name__)
 
 
 class MissingParameterException(Exception):
-    def __init__(self, parameters):
+
+    def __init__(self, parameters, *args, **kwargs):
         self.parameters = parameters
-        self.message = ("Missing required parameters: %s" %
-                        ', '.join(parameters))
+        message = 'Missing required parameters: %s' % (
+            ', '.join(parameters),
+        )
+        super(MissingParameterException, self).__init__(message, *args, **kwargs)
 
-    def __str__(self):
-        return self.message
+
+class ParameterDoesNotExist(Exception):
+
+    def __init__(self, parameter, *args, **kwargs):
+        message = 'Parameter: "%s" does not exist in output' % (parameter,)
+        super(ParameterDoesNotExist, self).__init__(message, *args, **kwargs)
 
 
 def get_stack_full_name(cfn_base, stack_name):
@@ -141,16 +145,21 @@ def conn(self):
             self._conn = ConnectionManager(self.region)
         return self._conn
 
+    @property
+    def s3_conn(self):
+        if not hasattr(self, '_s3_conn'):
+            self._s3_conn = ConnectionManager().s3
+        return self._s3_conn
+
     @property
     def cfn_bucket(self):
         if not getattr(self, '_cfn_bucket', None):
-            s3 = self.conn.s3
             try:
-                self._cfn_bucket = s3.get_bucket(self.bucket_name)
+                self._cfn_bucket = self.s3_conn.get_bucket(self.bucket_name)
             except S3ResponseError, e:
                 if e.error_code == 'NoSuchBucket':
                     logger.debug("Creating bucket %s.", self.bucket_name)
-                    self._cfn_bucket = s3.create_bucket(
+                    self._cfn_bucket = self.s3_conn.create_bucket(
                         self.bucket_name,
                         location=get_bucket_location(self.region))
                 elif e.error_code == 'AccessDenied':
@@ -260,8 +269,11 @@ def resolve_parameters(self, parameters, blueprint):
             if isinstance(value, basestring) and '::' in value:
                 # Get from the Output of another stack in the stack_map
                 stack_name, output = value.split('::')
-                self.get_outputs(stack_name)
-                value = self.outputs[stack_name][output]
+                stack_outputs = self.get_outputs(stack_name)
+                try:
+                    value = stack_outputs[output]
+                except KeyError:
+                    raise ParameterDoesNotExist(value)
             params[k] = value
         return params
 
@@ -288,6 +300,7 @@ def build_stack_tags(self, stack_context, template_url):
                 'stacker_namespace': self.namespace}
         if requires:
             tags['required_stacks'] = ':'.join(requires)
+        return tags
 
     def create_stack(self, full_name, template_url, parameters, tags):
         """ Creates a stack in CloudFormation """
@@ -353,7 +366,8 @@ def get_outputs(self, stack_name, force=False):
         Updates the local output cache with the values it finds.
         """
         if stack_name in self.outputs and not force:
-            return
+            return self.outputs[stack_name]
+
         logger.debug("Getting outputs from stack %s.", stack_name)
 
         full_name = self.get_stack_full_name(stack_name)
@@ -366,7 +380,7 @@ def get_outputs(self, stack_name, force=False):
         for output in stack.outputs:
             logger.debug("    %s: %s", output.key, output.value)
             stack_outputs[output.key] = output.value
-        return self.outputs
+        return self.outputs[stack_name]
 
     def sync_plan_status(self):
         """ Updates the status of each stack in the local plan.

stacker/plan.py

@@ -1,10 +1,12 @@
+import copy
+from collections import (
+    OrderedDict,
+    Iterable,
+)
 import logging
 
 logger = logging.getLogger(__name__)
 
-import copy
-from collections import OrderedDict, Iterable
-
 INPROGRESS_STATUSES = ('CREATE_IN_PROGRESS',
                        'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS',
                        'UPDATE_IN_PROGRESS')

stacker/tests/test_builder.py

@@ -1,8 +1,14 @@
-import unittest
 from collections import namedtuple
+import unittest
+
+import mock
 
 from stacker.builder import (
-    gather_parameters, handle_missing_parameters, MissingParameterException
+    Builder,
+    gather_parameters,
+    handle_missing_parameters,
+    MissingParameterException,
+    ParameterDoesNotExist,
 )
 
 
@@ -84,3 +90,21 @@ def test_stack_params_dont_override_given_params(self):
         required = ['Address']
         result = handle_missing_parameters(def_params, required, stack)
         self.assertEqual(result, def_params.items())
+
+
+class TestBuilder(unittest.TestCase):
+
+    def setUp(self):
+        self.builder = Builder('us-east-1', 'namespace')
+
+    def test_resolve_parameters_referencing_non_existant_output(self):
+        parameters = {
+            'param_1': 'mock::output_1',
+            'param_2': 'mock::does_not_exist',
+        }
+        with mock.patch.object(self.builder, 'get_outputs') as mock_outputs:
+            mock_outputs.return_value = {'output_1': 'output'}
+            mock_blueprint = mock.MagicMock()
+            type(mock_blueprint).parameters = parameters
+            with self.assertRaises(ParameterDoesNotExist):
+                self.builder.resolve_parameters(parameters, mock_blueprint)

stacker/util.py

@@ -1,14 +1,13 @@
+import importlib
 import logging
-
-logger = logging.getLogger(__name__)
-
-import time
 import re
-import importlib
 import sys
+import time
 
 from boto.route53.record import ResourceRecordSets
 
+logger = logging.getLogger(__name__)
+
 
 def retry_with_backoff(function, args=None, kwargs=None, attempts=5,
                        min_delay=1, max_delay=3, exc_list=None):
@@ -125,39 +124,40 @@ def handle_hooks(stage, hooks, region, namespace, mappings, parameters):
     These are pieces of code that we want to run before/after the builder
     builds the stacks.
     """
-    if hooks:
-        hook_paths = []
-        for i, h in enumerate(hooks):
-            try:
-                hook_paths.append(h['path'])
-            except KeyError:
-                raise ValueError("%s hook #%d missing path." % (stage, i))
-
-        logger.info("Executing %s hooks: %s", stage, ", ".join(hook_paths))
-        for hook in hooks:
-            required = hook.get('required', True)
-            kwargs = hook.get('args', {})
-            try:
-                method = load_object_from_string(hook['path'])
-            except (AttributeError, ImportError):
-                logger.exception("Unable to load method at %s:", hook['path'])
-                if required:
-                    raise
-                continue
-            try:
-                result = method(region, namespace, mappings, parameters,
-                                **kwargs)
-            except Exception:
-                logger.exception("Method %s threw an exception:", hook['path'])
-                if required:
-                    raise
-                continue
-            if not result:
-                if required:
-                    logger.error("Required hook %s failed. Return value: %s",
-                                 hook['path'], result)
-                    sys.exit(1)
-                logger.warning("Non-required hook %s failed. Return value: %s",
-                               hook['path'], result)
-    else:
+    if not hooks:
         logger.debug("No %s hooks defined.", stage)
+        return
+
+    hook_paths = []
+    for i, h in enumerate(hooks):
+        try:
+            hook_paths.append(h['path'])
+        except KeyError:
+            raise ValueError("%s hook #%d missing path." % (stage, i))
+
+    logger.info("Executing %s hooks: %s", stage, ", ".join(hook_paths))
+    for hook in hooks:
+        required = hook.get('required', True)
+        kwargs = hook.get('args', {})
+        try:
+            method = load_object_from_string(hook['path'])
+        except (AttributeError, ImportError):
+            logger.exception("Unable to load method at %s:", hook['path'])
+            if required:
+                raise
+            continue
+        try:
+            result = method(region, namespace, mappings, parameters,
+                            **kwargs)
+        except Exception:
+            logger.exception("Method %s threw an exception:", hook['path'])
+            if required:
+                raise
+            continue
+        if not result:
+            if required:
+                logger.error("Required hook %s failed. Return value: %s",
+                             hook['path'], result)
+                sys.exit(1)
+            logger.warning("Non-required hook %s failed. Return value: %s",
+                           hook['path'], result)