Merge pull request #214 from ministryofjustice/fils-dependency-implem…

…entation

A stack deletion implementation

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * Adds missing permission of ELB `Describe*` on Resource `*` to
   ec2 iam policies.
+* Fixes cloudformation vpc dependency deletion issues
 
 ## v0.10.0
 

bootstrap_cfn/config.py

@@ -257,12 +257,14 @@ def vpc(self):
                 Application=Ref("AWS::StackId"),
                 Network="Public",
             ),
+            DependsOn=vpc.title,
         )
 
         gw_attachment = VPCGatewayAttachment(
             "AttachGateway",
             VpcId=Ref(vpc),
             InternetGatewayId=Ref(igw),
+            DependsOn=igw.title
         )
 
         route_table = RouteTable(
@@ -507,7 +509,6 @@ def rds(self, template):
             DBSubnetGroupDescription="VPC Subnets"
         )
         resources.append(rds_subnet_group)
-
         database_sg = SecurityGroup(
             "DatabaseSG",
             SecurityGroupIngress=[
@@ -526,6 +527,12 @@ def rds(self, template):
             ],
             VpcId=Ref("VPC"),
             GroupDescription="SG for EC2 Access to RDS",
+            # translates to DependsOn=["AttachGateway"],
+            # but if not, ensure the tests have been updated too
+            DependsOn=[
+                r.title
+                for r in self._find_resources(
+                    template, "AWS::EC2::VPCGatewayAttachment")],
         )
         resources.append(database_sg)
 
@@ -741,6 +748,17 @@ def elb(self, template):
                 ),
                 Policies=elb_policies
             )
+
+            if elb['scheme'] == 'internet-facing':
+                vpc_gw_att = [
+                    r.title
+                    for r in self._find_resources(
+                        template, "AWS::EC2::VPCGatewayAttachment")]
+                # if a VPCGatewayAttachment exists in template make the ELB depend on it
+                # because ELB is public/internet facing and needs Internet Gateway
+                if vpc_gw_att:
+                    load_balancer.DependsOn = vpc_gw_att
+
             if "health_check" in elb:
                 load_balancer.HealthCheck = HealthCheck(**elb['health_check'])
 
@@ -1113,6 +1131,13 @@ def ec2(self):
             LaunchConfigurationName=Ref(launch_config),
             HealthCheckGracePeriod=health_check_grace_period,
             HealthCheckType=health_check_type,
+            # should be equivalent to DependsOn=["AttachGateway"],
+            # but if not, ensure the tests have been updated too
+            DependsOn=[
+                v.title
+                for v in self.vpc()
+                if 'AWS::EC2::VPCGatewayAttachment' == v.resource['Type']]
+
         )
         resources.append(scaling_group)
 

tests/tests.py

@@ -294,6 +294,14 @@ def test_custom_s3_policy(self):
         compare(expected_outputs, actual_outputs)
 
     def test_rds(self):
+
+        template = Template()
+        config = ConfigParser(
+            ProjectConfig(
+                'tests/sample-project.yaml',
+                'dev',
+                'tests/sample-project-passwords.yaml').config, 'my-stack-name')
+
         db_sg = ec2.SecurityGroup('DatabaseSG')
         db_sg.VpcId = Ref('VPC')
         db_sg.GroupDescription = 'SG for EC2 Access to RDS'
@@ -311,6 +319,7 @@ def test_rds(self):
              "IpProtocol": "tcp",
              "CidrIp": FindInMap("SubnetConfig", "VPC", "CIDR")}
         ]
+        db_sg.DependsOn = []
 
         db_subnet = rds.DBSubnetGroup('RDSSubnetGroup')
         db_subnet.SubnetIds = [Ref('SubnetA'), Ref('SubnetB'), Ref('SubnetC')]
@@ -336,26 +345,113 @@ def test_rds(self):
 
         known = [db_instance, db_subnet, db_sg]
 
+        config.rds(template)
+        resources = template.resources.values()
+        rds_dict = self._resources_to_dict(resources)
+        # RDS dict will contain DBIdentifier, which is random.
+        # So we check it seperately here then remove it
+        self.assertTrue("DBInstanceIdentifier" in rds_dict["RDSInstance"]["Properties"],
+                        "test_rds: template does not contain DBInstanceIdentifier")
+        identifier = rds_dict["RDSInstance"]["Properties"]["DBInstanceIdentifier"]
+        # Identifier can be optionally be defined in the yaml template for compatibility.
+        # We're only testing the case where it's defined. If left undefined AWS will
+        # generate a random one.
+        self.assertEquals(identifier, 'test-dev')
+        rds_dict["RDSInstance"]["Properties"].pop("DBInstanceIdentifier")
+        known = self._resources_to_dict(known)
+        compare(known, rds_dict)
+
+        # Test for outputs
+        expected_outputs = {
+            "dbhost": {
+                "Description": "RDS Hostname",
+                "Value": {"Fn::GetAtt": ["RDSInstance", "Endpoint.Address"]}
+            },
+            "dbport": {
+                "Description": "RDS Port",
+                "Value": {
+                    "Fn::GetAtt": ["RDSInstance", "Endpoint.Port"]
+                }
+            }
+        }
+        actual_outputs = self._resources_to_dict(template.outputs.values())
+        compare(expected_outputs, actual_outputs)
+
+    def test_rds_with_vpc_dependencies(self):
+        template = Template()
+
         config = ConfigParser(
             ProjectConfig(
                 'tests/sample-project.yaml',
                 'dev',
                 'tests/sample-project-passwords.yaml').config, 'my-stack-name')
+        # generate and add the VPCGatewayAttachment resource to the template
+        # to ensure it is passed as an attachment (DependsOn) later
+        vpc_resources_needed_for_rds = [
+            res for res in config.vpc()
+            if res.resource_type == 'AWS::EC2::VPCGatewayAttachment']
+        map(template.add_resource, vpc_resources_needed_for_rds)
+
+        db_sg = ec2.SecurityGroup('DatabaseSG')
+        db_sg.VpcId = Ref('VPC')
+        db_sg.GroupDescription = 'SG for EC2 Access to RDS'
+        db_sg.SecurityGroupIngress = [
+            {"ToPort": 5432,
+             "FromPort": 5432,
+             "IpProtocol": "tcp",
+             "CidrIp": FindInMap("SubnetConfig", "VPC", "CIDR")},
+            {"ToPort": 1433,
+             "FromPort": 1433,
+             "IpProtocol": "tcp",
+             "CidrIp": FindInMap("SubnetConfig", "VPC", "CIDR")},
+            {"ToPort": 3306,
+             "FromPort": 3306,
+             "IpProtocol": "tcp",
+             "CidrIp": FindInMap("SubnetConfig", "VPC", "CIDR")}
+        ]
+        db_sg.DependsOn = ["AttachGateway"]
+
+        db_subnet = rds.DBSubnetGroup('RDSSubnetGroup')
+        db_subnet.SubnetIds = [Ref('SubnetA'), Ref('SubnetB'), Ref('SubnetC')]
+        db_subnet.DBSubnetGroupDescription = 'VPC Subnets'
+
+        db_instance = rds.DBInstance('RDSInstance', DependsOn=db_sg.title)
+        db_instance.MultiAZ = False
+        db_instance.MasterUsername = 'testuser'
+        db_instance.MasterUserPassword = 'testpassword'
+        db_instance.DBName = 'test'
+        db_instance.PubliclyAccessible = False
+        db_instance.StorageEncrypted = False
+        db_instance.StorageType = 'gp2'
+        db_instance.AllocatedStorage = 5
+        db_instance.AllowMajorVersionUpgrade = False
+        db_instance.AutoMinorVersionUpgrade = False
+        db_instance.BackupRetentionPeriod = 1
+        db_instance.DBInstanceClass = 'db.t2.micro'
+        db_instance.Engine = 'postgres'
+        db_instance.EngineVersion = '9.3.5'
+        db_instance.VPCSecurityGroups = [GetAtt(db_sg, 'GroupId')]
+        db_instance.DBSubnetGroupName = Ref(db_subnet)
+
+        known = [db_instance, db_subnet, db_sg]
 
-        template = Template()
         config.rds(template)
         resources = template.resources.values()
         rds_dict = self._resources_to_dict(resources)
         # RDS dict will contain DBIdentifier, which is random.
         # So we check it seperately here then remove it
         self.assertTrue("DBInstanceIdentifier" in rds_dict["RDSInstance"]["Properties"],
-                        "test_rds: template does not contain DBInstanceIdentifier")
+                        "test_rds_with_vpc_dependencies: "
+                        "template does not contain DBInstanceIdentifier")
         identifier = rds_dict["RDSInstance"]["Properties"]["DBInstanceIdentifier"]
         # Identifier can be optionally be defined in the yaml template for compatibility.
         # We're only testing the case where it's defined. If left undefined AWS will
         # generate a random one.
         self.assertEquals(identifier, 'test-dev')
         rds_dict["RDSInstance"]["Properties"].pop("DBInstanceIdentifier")
+        # keep just the keys (rds) we want to compare,
+        # we are done with the vpc so pop the vpc gw attachment
+        [rds_dict.pop(res.title) for res in vpc_resources_needed_for_rds]
         known = self._resources_to_dict(known)
         compare(known, rds_dict)
 
@@ -1325,7 +1421,8 @@ def test_ec2(self):
             LaunchConfigurationName=Ref("BaseHostLaunchConfig"),
             AvailabilityZones=GetAZs(""),
             HealthCheckGracePeriod=300,
-            HealthCheckType='EC2'
+            HealthCheckType='EC2',
+            DependsOn=['AttachGateway']
         )
 
         BaseHostSG = SecurityGroup(