Merge pull request #209 from ministryofjustice/set_rds_defaults

Use a configuration file for default cloudformations settings

bootstrap_cfn/config.py

@@ -28,21 +28,80 @@ class ProjectConfig:
 
     config = None
 
-    def __init__(self, config, environment, passwords=None):
+    def __init__(self,
+                 config,
+                 environment,
+                 passwords=None,
+                 defaults=os.path.join(os.path.dirname(__file__), 'config_defaults.yaml')):
         try:
-            self.config = self.load_yaml(config)[environment]
+            self.config = {}
+            # Load all the necessary config files and defaults
+            config_defaults = self.load_yaml(defaults).get(environment,
+                                                           self.load_yaml(defaults)['default'])
+            user_config = self.load_yaml(config)[environment]
+            passwords_config = {}
+            if passwords:
+                passwords_config = self.load_yaml(passwords).get(environment, {})
+
+            # Validate all the settings we have loaded in
+            logging.info('bootstrap-cfn:: Validating default settings for environment %s in file %s'
+                         % (environment, defaults))
+            self.validate_configuration_settings(config_defaults)
+            logging.info('bootstrap-cfn:: Validating user settings for environment %s in file %s'
+                         % (environment, config))
+            self.validate_configuration_settings(user_config)
+            logging.info('bootstrap-cfn:: Validating passwords settings for environment %s in file %s'
+                         % (environment, passwords))
+            self.validate_configuration_settings(passwords_config)
+
+            # Collect together all the config keys the user has specified
+            all_user_config_keys = set(user_config.keys()) | set(passwords_config.keys())
+
+            # Only set configuration settings where we have specified that component in the user config
+            # This means we only get non-required components RDS, elasticache, etc if we have requested them
+            for config_key in all_user_config_keys:
+                # we're going to merge in order of,
+                # defaults <- user_config <- secrets_config
+
+                # Catch badly formatted yaml where we get NoneType values,
+                # merging in these will overwrite all the other config
+                self.config[config_key] = config_defaults.get(config_key, {})
+                # Overwrite defaults with user_config values
+                self.config[config_key] = utils.dict_merge(self.config[config_key],
+                                                           user_config.get(config_key, {}))
+                # Overwrite user config with password config values
+                self.config[config_key] = utils.dict_merge(self.config[config_key],
+                                                           passwords_config.get(config_key, {}))
         except KeyError:
             raise errors.BootstrapCfnError("Environment " + environment + " not found")
 
-        if passwords:
-            passwords_dict = self.load_yaml(passwords)[environment]
-            self.config = utils.dict_merge(self.config, passwords_dict)
-
     @staticmethod
     def load_yaml(fp):
         if os.path.exists(fp):
             return yaml.load(open(fp).read())
 
+    @staticmethod
+    def validate_configuration_settings(configuration_settings):
+        """
+        Run some sanity checks on the configuration settings we're going to use
+
+        Args:
+            config: The settings object we want to validate
+
+        Raises:
+            CfnConfigError
+        """
+        # Basic settings checks
+        # settings should be a dictionary
+        if not isinstance(configuration_settings, dict):
+            raise errors.CfnConfigError("Configuration settings are not in dictionary format")
+
+        for key, value in configuration_settings.iteritems():
+            # No base keys should have a None value
+            if value is None:
+                raise errors.CfnConfigError("Configuration key value %s is None."
+                                            % (key))
+
 
 class ConfigParser(object):
 
@@ -261,20 +320,26 @@ def iam(self):
                 }]
             },
         )
+        # Set required policy actions
+        policy_actions = [{"Action": ["autoscaling:Describe*"], "Resource": "*", "Effect": "Allow"},
+                          {"Action": ["cloudformation:Describe*"], "Resource": "*", "Effect": "Allow"}]
+
+        # Only define policy actions if the components are enabled in the config
+        if 'ec2' in self.data:
+            policy_actions.append({"Action": ["ec2:Describe*"], "Resource": "*", "Effect": "Allow"})
+            policy_actions.append({"Action": ["ec2:CreateTags"], "Resource": "*", "Effect": "Allow"})
+        if 'rds' in self.data:
+            policy_actions.append({"Action": ["rds:Describe*"], "Resource": "*", "Effect": "Allow"})
+        if 'elasticache' in self.data:
+            policy_actions.append({"Action": ["elasticloadbalancing:Describe*"], "Resource": "*", "Effect": "Allow"})
+            policy_actions.append({"Action": ["elasticache:Describe*"], "Resource": "*", "Effect": "Allow"})
+        if 's3' in self.data:
+            policy_actions.append({"Action": ["s3:List*"], "Resource": "*", "Effect": "Allow"})
 
         role_policies = PolicyType(
             "RolePolicies",
             PolicyName="BaseHost",
-            PolicyDocument={"Statement": [
-                {"Action": ["autoscaling:Describe*"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["ec2:Describe*"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["ec2:CreateTags"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["rds:Describe*"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["elasticloadbalancing:Describe*"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["elasticache:Describe*"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["cloudformation:Describe*"], "Resource": "*", "Effect": "Allow"},
-                {"Action": ["s3:List*"], "Resource": "*", "Effect": "Allow"}
-            ]},
+            PolicyDocument={"Statement": policy_actions},
             Roles=[Ref(role)],
         )
         instance_profile = InstanceProfile(
@@ -417,18 +482,18 @@ def rds(self, template):
         # REQUIRED FIELDS MAPPING
         required_fields = {
             'db-name': 'DBName',
+            'db-master-username': 'MasterUsername',
+            'db-master-password': 'MasterUserPassword',
+        }
+
+        optional_fields = {
             'storage': 'AllocatedStorage',
             'storage-type': 'StorageType',
             'backup-retention-period': 'BackupRetentionPeriod',
-            'db-master-username': 'MasterUsername',
-            'db-master-password': 'MasterUserPassword',
             'db-engine': 'Engine',
             'db-engine-version': 'EngineVersion',
             'instance-class': 'DBInstanceClass',
-            'multi-az': 'MultiAZ'
-        }
-
-        optional_fields = {
+            'multi-az': 'MultiAZ',
             'storage-encrypted': 'StorageEncrypted',
             'identifier': 'DBInstanceIdentifier'
         }
@@ -470,7 +535,6 @@ def rds(self, template):
             AutoMinorVersionUpgrade=False,
             VPCSecurityGroups=[GetAtt(database_sg, "GroupId")],
             DBSubnetGroupName=Ref(rds_subnet_group),
-            StorageEncrypted=True,
             DependsOn=database_sg.title
         )
         resources.append(rds_instance)
@@ -524,20 +588,8 @@ def elasticache(self, template):
             'port': 'Port',
         }
 
-        # Setup params and config
-        component_config = self.data['elasticache']
-        # Setup defaults
-        if 'clusters' not in component_config:
-            component_config['clusters'] = 3
-        if 'node_type' not in component_config:
-            component_config['node_type'] = 'cache.m1.small'
-        if 'port' not in component_config:
-            component_config['port'] = 6379
-
-        engine = 'redis'
-
         # Generate snapshot arns
-        seeds = component_config.get('seeds', None)
+        seeds = self.data['elasticache'].get('seeds', None)
         snapshot_arns = []
         if seeds:
             # Get s3 seeds
@@ -551,8 +603,8 @@ def elasticache(self, template):
         es_sg = SecurityGroup(
             "ElasticacheSG",
             SecurityGroupIngress=[
-                {"ToPort": component_config['port'],
-                 "FromPort": component_config['port'],
+                {"ToPort": self.data['elasticache']['port'],
+                 "FromPort": self.data['elasticache']['port'],
                  "IpProtocol": "tcp",
                  "CidrIp": FindInMap("SubnetConfig", "VPC", "CIDR")}
             ],
@@ -571,9 +623,9 @@ def elasticache(self, template):
         elasticache_replication_group = ReplicationGroup(
             "ElasticacheReplicationGroup",
             ReplicationGroupDescription='Elasticache Replication Group',
-            Engine=engine,
-            NumCacheClusters=component_config['clusters'],
-            CacheNodeType=component_config['node_type'],
+            Engine=self.data['elasticache'].get('engine'),
+            NumCacheClusters=self.data['elasticache']['clusters'],
+            CacheNodeType=self.data['elasticache']['node_type'],
             SecurityGroupIds=[GetAtt(es_sg, "GroupId")],
             CacheSubnetGroupName=Ref(es_subnet_group),
             SnapshotArns=snapshot_arns
@@ -582,15 +634,15 @@ def elasticache(self, template):
 
         # TEST FOR REQUIRED FIELDS AND EXIT IF MISSING ANY
         for yaml_key, prop in required_fields.iteritems():
-            if yaml_key not in component_config:
+            if yaml_key not in self.data['elasticache']:
                 print "\n\n[ERROR] Missing Elasticache fields [%s]" % yaml_key
                 sys.exit(1)
             else:
-                elasticache_replication_group.__setattr__(prop, component_config[yaml_key])
+                elasticache_replication_group.__setattr__(prop, self.data['elasticache'][yaml_key])
 
         for yaml_key, prop in optional_fields.iteritems():
-            if yaml_key in component_config:
-                elasticache_replication_group.__setattr__(prop, component_config[yaml_key])
+            if yaml_key in self.data['elasticache']:
+                elasticache_replication_group.__setattr__(prop, self.data['elasticache'][yaml_key])
 
         # Add resources and outputs
         map(template.add_resource, resources)
@@ -603,7 +655,7 @@ def elasticache(self, template):
         template.add_output(Output(
             "ElasticacheEngine",
             Description="Elasticache Engine",
-            Value=engine
+            Value=self.data['elasticache'].get('engine')
         ))
 
     def elb(self, template):

bootstrap_cfn/config_defaults.yaml

@@ -0,0 +1,73 @@
+# Production is secure by default and may have high resource specifications
+prod:
+  ec2: &prod_ec2
+    auto_scaling: &prod_ec2_auto_scaling
+      desired: 2
+      max: 5
+      min: 1
+    parameters: &prod_ec2_parameters
+      KeyName: default
+      InstanceType: t2.micro
+    block_devices: &prod_ec2_block_devices
+      - DeviceName: /dev/sda1
+        VolumeSize: 20
+  rds: &prod_rds
+    storage: 5
+    storage-type: 'gp2'
+    backup-retention-period: 30
+    db-engine: 'postgres'
+    db-engine-version: '9.3.5'
+    instance-class: 'db.t2.large'
+    multi-az: True
+    storage-encrypted: True
+  elasticache: &prod_elasticache
+    clusters: 3
+    node_type: 'cache.m1.small'
+    port: 6379
+    engine: 'redis'
+
+# Dev is not secure by default and has lower powered resource specifications
+dev:
+  ec2: &dev_ec2
+    auto_scaling: &dev_ec2_auto_scaling
+      desired: 1
+      max: 3
+      min: 1
+    parameters: &dev_ec2_parameters
+      KeyName: default
+      InstanceType: t2.micro
+    block_devices: &dev_ec2_block_devices
+      - DeviceName: /dev/sda1
+        VolumeSize: 20
+  rds: &dev_rds
+    storage: 5
+    storage-type: 'gp2'
+    backup-retention-period: 1
+    db-engine: 'postgres'
+    db-engine-version: '9.3.5'
+    instance-class: 'db.t2.small'
+    multi-az: False
+    storage-encrypted: False
+  elasticache: &dev_elasticache
+    clusters: 3
+    node_type: 'cache.m1.small'
+    port: 6379
+    engine: 'redis'
+
+# Staging should be equivalent to prod
+staging:
+  ec2:
+    <<: *prod_ec2
+  rds:
+    <<: *prod_rds
+  elasticache:
+    <<: *prod_elasticache
+
+# The default should be the most secure
+default:
+  ec2:
+    <<: *prod_ec2
+  rds:
+    <<: *prod_rds
+  elasticache:
+    <<: *prod_elasticache

setup.py

@@ -12,7 +12,8 @@
     description='MOJDS cloudformation bootstrap tool',
     long_description="",
     packages=find_packages(exclude=["tests"]),
-    package_data={'bootstrap_cfn': ['stacks/*']},
+    include_package_data=True,
+    package_data={'bootstrap_cfn': ['config_defaults.yaml', 'stacks/*']},
     zip_safe=False,
     platforms='any',
     test_suite='tests',

tests/cloudformation/sample-project_minimal-secrets.yaml

@@ -0,0 +1,3 @@
+prod:
+  rds:
+    db-master-password: testpassword

tests/cloudformation/sample-project_minimal.yaml

@@ -0,0 +1,50 @@
+prod:
+  ec2:
+    tags:
+      Apps: test
+    security_groups:
+      AnotherSG:
+        - IpProtocol: tcp
+          FromPort: 443
+          ToPort: 443
+          SourceSecurityGroupName:
+            Ref: BaseHostSG
+      BaseHostSG:
+        - IpProtocol: tcp
+          FromPort: 22
+          ToPort: 22
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 80
+          ToPort: 80
+          CidrIp: 0.0.0.0/0
+  elb:
+    - name: test-dev-external
+      hosted_zone: kyrtest.pf.dsd.io.
+      scheme: internet-facing
+      listeners:
+        - LoadBalancerPort: 80
+          InstancePort: 80
+          Protocol: TCP
+        - LoadBalancerPort: 443
+          InstancePort: 443
+          Protocol: TCP
+  s3: {}
+  rds:
+    db-name: test
+    db-master-username: testuser
+  elasticache: {}
+  ssl:
+    my-cert:
+      cert: |
+        -----BEGIN CERTIFICATE-----
+        blahblahblah
+        -----END CERTIFICATE-----
+      key: |
+        -----BEGIN RSA PRIVATE KEY-----
+        blahblahblah
+        -----END RSA PRIVATE KEY-----
+      chain: |
+        -----BEGIN CERTIFICATE-----
+        blahblahblah
+        -----END CERTIFICATE-----