Use a configuration file for default cloudformations settings

Currently we have some default settings coded into config.py.
This change introduces the use of a configuration file to
specify default settings for cloudformation. These settings are
replaced by those placed into the project configuration files.

In particular, this means that we can provide defaults for each
environments seperately, putting less resources on dev environments
than on prod. In this case we need substantial RDS instance types
in prod to support defaulting to encryption. This is replicated
across all environments. With this change we can set the dev
environment to use no encryption and small instances as default.

bootstrap_cfn/config.py

@@ -28,21 +28,88 @@ class ProjectConfig:
 
     config = None
 
-    def __init__(self, config, environment, passwords=None):
+    def __init__(self,
+                 config,
+                 environment,
+                 passwords=None,
+                 defaults=os.path.join(os.path.dirname(__file__), 'config_defaults.yaml')):
         try:
-            self.config = self.load_yaml(config)[environment]
+            self.config = {}
+            # Load all the necessary config files and defaults
+            config_defaults = self.load_yaml(defaults).get(environment,
+                                                           self.load_yaml(defaults)['default'])
+            user_config = self.load_yaml(config)[environment]
+            passwords_config = {}
+            if passwords:
+                passwords_config = self.load_yaml(passwords).get(environment, {})
+
+            # Validate all the settings we have loaded in
+            logging.info('bootstrap-cfn:: Validating default settings for environment %s in file %s'
+                         % (environment, defaults))
+            self.validate_configuration_settings(config_defaults)
+            logging.info('bootstrap-cfn:: Validating user settings for environment %s in file %s'
+                         % (environment, config))
+            self.validate_configuration_settings(user_config)
+            logging.info('bootstrap-cfn:: Validating passwords settings for environment %s in file %s'
+                         % (environment, passwords))
+            self.validate_configuration_settings(passwords_config)
+
+            # Collect together all the config keys the user has specified
+            all_user_config_keys = set(user_config.keys()) | set(passwords_config.keys())
+
+            # Only set configuration settings where we have specified that component in the user config
+            # This means we only get non-required components RDS, elasticache, etc if we have requested them
+            for config_key in all_user_config_keys:
+                # we're going to merge in order of,
+                # defaults <- user_config <- secrets_config
+
+                # Catch badly formatted yaml where we get NoneType values,
+                # merging in these will overwrite all the other config
+                config_defaults_value = config_defaults.get(config_key, {})
+                if config_defaults_value is None:
+                    raise errors.CfnConfigError("Defaults config value '%s' in None"
+                                                % (config_key))
+                self.config[config_key] = config_defaults_value
+
+                # Overwrite defaults with user_config values
+                user_config_value = user_config.get(config_key, {})
+                if user_config_value is None:
+                    raise errors.CfnConfigError("User config value '%s' in None"
+                                                % (config_key))
+                self.config[config_key] = utils.dict_merge(self.config[config_key],
+                                                           user_config_value)
+
+                # Overwrite user config with password config values
+                passwords_config_value = passwords_config.get(config_key, {})
+                if passwords_config_value is None:
+                    raise errors.CfnConfigError("Passwords config value '%s' in None"
+                                                % (config_key))
+                self.config[config_key] = utils.dict_merge(self.config[config_key],
+                                                           passwords_config_value)
         except KeyError:
             raise errors.BootstrapCfnError("Environment " + environment + " not found")
 
-        if passwords:
-            passwords_dict = self.load_yaml(passwords)[environment]
-            self.config = utils.dict_merge(self.config, passwords_dict)
-
     @staticmethod
     def load_yaml(fp):
         if os.path.exists(fp):
             return yaml.load(open(fp).read())
 
+    @staticmethod
+    def validate_configuration_settings(configuration_settings):
+        """
+        Run some sanity checks on the configuration settings we're going to use
+
+        Args:
+            config: The settings object we want to validate
+
+        Raises:
+            CfnConfigError
+        """
+        # Basic settings checks
+        # settings should be a dictionary
+        if not isinstance(configuration_settings, dict):
+            raise errors.CfnConfigError("Configuration settings are not in dictionary format")
+
 
 class ConfigParser(object):
 
@@ -417,18 +484,18 @@ def rds(self, template):
         # REQUIRED FIELDS MAPPING
         required_fields = {
             'db-name': 'DBName',
+            'db-master-username': 'MasterUsername',
+            'db-master-password': 'MasterUserPassword',
+        }
+
+        optional_fields = {
             'storage': 'AllocatedStorage',
             'storage-type': 'StorageType',
             'backup-retention-period': 'BackupRetentionPeriod',
-            'db-master-username': 'MasterUsername',
-            'db-master-password': 'MasterUserPassword',
             'db-engine': 'Engine',
             'db-engine-version': 'EngineVersion',
             'instance-class': 'DBInstanceClass',
-            'multi-az': 'MultiAZ'
-        }
-
-        optional_fields = {
+            'multi-az': 'MultiAZ',
             'storage-encrypted': 'StorageEncrypted',
             'identifier': 'DBInstanceIdentifier'
         }
@@ -470,7 +537,6 @@ def rds(self, template):
             AutoMinorVersionUpgrade=False,
             VPCSecurityGroups=[GetAtt(database_sg, "GroupId")],
             DBSubnetGroupName=Ref(rds_subnet_group),
-            StorageEncrypted=True,
             DependsOn=database_sg.title
         )
         resources.append(rds_instance)
@@ -524,20 +590,8 @@ def elasticache(self, template):
             'port': 'Port',
         }
 
-        # Setup params and config
-        component_config = self.data['elasticache']
-        # Setup defaults
-        if 'clusters' not in component_config:
-            component_config['clusters'] = 3
-        if 'node_type' not in component_config:
-            component_config['node_type'] = 'cache.m1.small'
-        if 'port' not in component_config:
-            component_config['port'] = 6379
-
-        engine = 'redis'
-
         # Generate snapshot arns
-        seeds = component_config.get('seeds', None)
+        seeds = self.data['elasticache'].get('seeds', None)
         snapshot_arns = []
         if seeds:
             # Get s3 seeds
@@ -551,8 +605,8 @@ def elasticache(self, template):
         es_sg = SecurityGroup(
             "ElasticacheSG",
             SecurityGroupIngress=[
-                {"ToPort": component_config['port'],
-                 "FromPort": component_config['port'],
+                {"ToPort": self.data['elasticache']['port'],
+                 "FromPort": self.data['elasticache']['port'],
                  "IpProtocol": "tcp",
                  "CidrIp": FindInMap("SubnetConfig", "VPC", "CIDR")}
             ],
@@ -571,9 +625,9 @@ def elasticache(self, template):
         elasticache_replication_group = ReplicationGroup(
             "ElasticacheReplicationGroup",
             ReplicationGroupDescription='Elasticache Replication Group',
-            Engine=engine,
-            NumCacheClusters=component_config['clusters'],
-            CacheNodeType=component_config['node_type'],
+            Engine=self.data['elasticache'].get('engine'),
+            NumCacheClusters=self.data['elasticache']['clusters'],
+            CacheNodeType=self.data['elasticache']['node_type'],
             SecurityGroupIds=[GetAtt(es_sg, "GroupId")],
             CacheSubnetGroupName=Ref(es_subnet_group),
             SnapshotArns=snapshot_arns
@@ -582,15 +636,15 @@ def elasticache(self, template):
 
         # TEST FOR REQUIRED FIELDS AND EXIT IF MISSING ANY
         for yaml_key, prop in required_fields.iteritems():
-            if yaml_key not in component_config:
+            if yaml_key not in self.data['elasticache']:
                 print "\n\n[ERROR] Missing Elasticache fields [%s]" % yaml_key
                 sys.exit(1)
             else:
-                elasticache_replication_group.__setattr__(prop, component_config[yaml_key])
+                elasticache_replication_group.__setattr__(prop, self.data['elasticache'][yaml_key])
 
         for yaml_key, prop in optional_fields.iteritems():
-            if yaml_key in component_config:
-                elasticache_replication_group.__setattr__(prop, component_config[yaml_key])
+            if yaml_key in self.data['elasticache']:
+                elasticache_replication_group.__setattr__(prop, self.data['elasticache'][yaml_key])
 
         # Add resources and outputs
         map(template.add_resource, resources)
@@ -603,7 +657,7 @@ def elasticache(self, template):
         template.add_output(Output(
             "ElasticacheEngine",
             Description="Elasticache Engine",
-            Value=engine
+            Value=self.data['elasticache'].get('engine')
         ))
 
     def elb(self, template):

bootstrap_cfn/config_defaults.yaml

@@ -0,0 +1,73 @@
+# Production is secure by default and may have high resource specifications
+prod:
+  ec2: &prod_ec2
+    auto_scaling: &prod_ec2_auto_scaling
+      desired: 2
+      max: 5
+      min: 1
+    parameters: &prod_ec2_parameters
+      KeyName: default
+      InstanceType: t2.micro
+    block_devices: &prod_ec2_block_devices
+      - DeviceName: /dev/sda1
+        VolumeSize: 20
+  rds: &prod_rds
+    storage: 5
+    storage-type: 'gp2'
+    backup-retention-period: 1
+    db-engine: 'postgres'
+    db-engine-version: '9.3.5'
+    instance-class: 'db.t2.large'
+    multi-az: True
+    storage-encrypted: True
+  elasticache: &prod_elasticache
+    clusters: 3
+    node_type: 'cache.m1.small'
+    port: 6379
+    engine: 'redis'
+
+# Dev is not secure by default and has lower powered resource specifications
+dev:
+  ec2: &dev_ec2
+    auto_scaling: &dev_ec2_auto_scaling
+      desired: 1
+      max: 3
+      min: 1
+    parameters: &dev_ec2_parameters
+      KeyName: default
+      InstanceType: t2.micro
+    block_devices: &dev_ec2_block_devices
+      - DeviceName: /dev/sda1
+        VolumeSize: 20
+  rds: &dev_rds
+    storage: 5
+    storage-type: 'gp2'
+    backup-retention-period: 1
+    db-engine: 'postgres'
+    db-engine-version: '9.3.5'
+    instance-class: 'db.t2.small'
+    multi-az: False
+    storage-encrypted: False
+  elasticache: &dev_elasticache
+    clusters: 3
+    node_type: 'cache.m1.small'
+    port: 6379
+    engine: 'redis'
+
+# Staging should be equivalent to prod
+staging:
+  ec2:
+    <<: *prod_ec2
+  rds:
+    <<: *prod_rds
+  elasticache:
+    <<: *prod_elasticache
+
+# The default should be the most secure
+default:
+  ec2:
+    <<: *prod_ec2
+  rds:
+    <<: *prod_rds
+  elasticache:
+    <<: *prod_elasticache

setup.py

@@ -12,7 +12,8 @@
     description='MOJDS cloudformation bootstrap tool',
     long_description="",
     packages=find_packages(exclude=["tests"]),
-    package_data={'bootstrap_cfn': ['stacks/*']},
+    include_package_data=True,
+    package_data={'bootstrap_cfn': ['config_defaults.yaml', 'stacks/*']},
     zip_safe=False,
     platforms='any',
     test_suite='tests',

tests/tests.py

@@ -281,7 +281,7 @@ def test_rds(self):
         db_instance.MasterUserPassword = 'testpassword'
         db_instance.DBName = 'test'
         db_instance.PubliclyAccessible = False
-        db_instance.StorageEncrypted = True
+        db_instance.StorageEncrypted = False
         db_instance.StorageType = 'gp2'
         db_instance.AllocatedStorage = 5
         db_instance.AllowMajorVersionUpgrade = False
@@ -1271,8 +1271,8 @@ def test_ec2(self):
 
         tags = [
             ('Name', {u'Fn::Join': [u'', [{u'Ref': u'AWS::StackName'}, u'-', u'ec2']]}),
-            ('Role', 'docker'),
             ('Apps', 'test'),
+            ('Role', 'docker'),
         ]
         ScalingGroup = AutoScalingGroup(
             "ScalingGroup",