Allow security groups in YAML to reference themselves

This means you can create a security group that allows access to (say)
port 9300 to other members of the same security group.

Prime example of this is Elasticsearch where anyone in the ES sg should
be able to speak to every other instance in the SG to form a cluster

CHANGELOG.md

@@ -1,6 +1,11 @@
 ## Version 0.5.x
+
 * Add an SSL cipher list policy pindown: implicitly (no YAML entry needed)
   or explicitly (with YAML entry)
+* Allow security groups in YAML to reference themselves (for example an
+  "elasticsearch" SG that allows other instances of that sg (and itself) to
+  access port 9300)
+
 ## Version 0.5.3
 
 * Improve message content when cfn_create raises an exception and fails.

bootstrap_cfn/config.py

@@ -10,7 +10,7 @@
     AutoScalingGroup, BlockDeviceMapping, EBSBlockDevice, Tag
 from troposphere.elasticloadbalancing import LoadBalancer, HealthCheck, \
     ConnectionDrainingPolicy, Policy
-from troposphere.ec2 import SecurityGroup
+from troposphere.ec2 import SecurityGroup, SecurityGroupIngress
 from troposphere.route53 import RecordSetGroup, RecordSet, AliasTarget
 from troposphere.ec2 import Route, Subnet, InternetGateway, VPC, \
     VPCGatewayAttachment, SubnetRouteTableAssociation, RouteTable
@@ -521,6 +521,19 @@ def elb(self):
                 elb_sgs.append(sg)
         return elb_list, elb_sgs
 
+    def _convert_ref_dict_to_objects(self, o):
+        """
+        Some troposphere objects need troposphere.Ref objects instead of a
+        plain dict of {"Ref": "x" }. This helper function will do such
+        transformations and return a new dict
+        """
+        def ref_fixup(x):
+            if isinstance(x, dict) and "Ref" in x:
+                return Ref(x["Ref"])
+            else:
+                return x
+        return dict([(k, ref_fixup(v)) for k, v in o.items()])
+
     def ec2(self):
         # LOAD STACK TEMPLATE
         data = self.data['ec2']
@@ -532,11 +545,34 @@ def ec2(self):
                 sg_name,
                 VpcId=Ref("VPC"),
                 GroupDescription="BaseHost Security Group",
-                SecurityGroupIngress=ingress
             )
+
             sgs.append(sg)
             resources.append(sg)
 
+            # Because we want to be able to add ingress rules to a security
+            # group that referes to itself (for example allow all instances in
+            # the sg to speak to each other on 9300 for Elasticsearch
+            # clustering) we create the SG in one resource and rules as other
+            # resources
+            #
+            # The yaml for this case is:
+            #
+            # security_groups:
+            #   EScluster:
+            #     - FromPort: 9300
+            #     - ToPort: 9300
+            #     - SourceSecurityGroupId: { Ref: EScluster }
+            for idx, rule in enumerate(ingress):
+                # Convert { Ref: "x"} to Ref("x")
+                rule = self._convert_ref_dict_to_objects(rule)
+
+                ingress = SecurityGroupIngress(
+                    "{}Rule{}".format(sg_name, idx),
+                    GroupId=Ref(sg),
+                    **rule)
+                resources.append(ingress)
+
         devices = []
         try:
             for i in data['block_devices']:

tests/tests.py

@@ -6,7 +6,7 @@
 from troposphere import iam, s3, rds, ec2
 from troposphere import Base64, FindInMap, GetAtt, GetAZs, Join
 from troposphere.autoscaling import AutoScalingGroup, Tag
-from troposphere.ec2 import SecurityGroup
+from troposphere.ec2 import SecurityGroup, SecurityGroupIngress
 from troposphere.autoscaling import LaunchConfiguration
 
 from troposphere.route53 import RecordSetGroup
@@ -486,8 +486,8 @@ def test_process(self):
         cfn_template = json.loads(config.process())
 
         wanted = [
-            "AnotherSG", "AttachGateway", "BaseHostLaunchConfig",
-            "BaseHostRole", "BaseHostSG", "DNStestdevexternal",
+            "AnotherSG", "AnotherSGRule0", "AttachGateway", "BaseHostLaunchConfig",
+            "BaseHostRole", "BaseHostSG", "BaseHostSGRule0", "BaseHostSGRule1", "DNStestdevexternal",
             "DNStestdevinternal", "DatabaseSG", "DefaultSGtestdevexternal",
             "DefaultSGtestdevinternal", "ELBtestdevexternal",
             "ELBtestdevinternal", "InstanceProfile", "InternetGateway",
@@ -542,8 +542,8 @@ def test_process_with_vpc_config(self):
         cfn_template = json.loads(config.process())
 
         wanted = [
-            "AnotherSG", "AttachGateway", "BaseHostLaunchConfig",
-            "BaseHostRole", "BaseHostSG", "DNStestdevexternal",
+            "AnotherSG", "AnotherSGRule0", "AttachGateway", "BaseHostLaunchConfig",
+            "BaseHostRole", "BaseHostSG", "BaseHostSGRule0", "BaseHostSGRule1", "DNStestdevexternal",
             "DNStestdevinternal", "DatabaseSG", "DefaultSGtestdevexternal",
             "DefaultSGtestdevinternal", "ELBtestdevexternal",
             "ELBtestdevinternal", "InstanceProfile", "InternetGateway",
@@ -957,23 +957,25 @@ def test_ec2(self):
 
         BaseHostSG = SecurityGroup(
             "BaseHostSG",
-            SecurityGroupIngress=[
-                {
-                    "ToPort": 22,
-                    "FromPort": 22,
-                    "IpProtocol": "tcp",
-                    "CidrIp": "0.0.0.0/0"
-                },
-                {
-                    "ToPort": 80,
-                    "FromPort": 80,
-                    "IpProtocol": "tcp",
-                    "CidrIp": "0.0.0.0/0"
-                }
-            ],
             VpcId=Ref("VPC"),
             GroupDescription="BaseHost Security Group",
         )
+        BaseHostSGRule0 = SecurityGroupIngress(
+            "BaseHostSGRule0",
+            ToPort=22,
+            FromPort=22,
+            CidrIp="0.0.0.0/0",
+            IpProtocol="tcp",
+            GroupId=Ref(BaseHostSG),
+        )
+        BaseHostSGRule1 = SecurityGroupIngress(
+            "BaseHostSGRule1",
+            ToPort=80,
+            FromPort=80,
+            CidrIp="0.0.0.0/0",
+            IpProtocol="tcp",
+            GroupId=Ref(BaseHostSG),
+        )
 
         BaseHostLaunchConfig = LaunchConfiguration(
             "BaseHostLaunchConfig",
@@ -998,18 +1000,18 @@ def test_ec2(self):
 
         AnotherSG = SecurityGroup(
             "AnotherSG",
-            SecurityGroupIngress=[
-                {
-                    "ToPort": 443,
-                    "FromPort": 443,
-                    "SourceSecurityGroupName": Ref(BaseHostSG),
-                    "IpProtocol": "tcp"
-                }
-            ],
             VpcId=Ref("VPC"),
             GroupDescription="BaseHost Security Group",
         )
-        known = [AnotherSG, BaseHostLaunchConfig, BaseHostSG, ScalingGroup]
+        AnotherSGRule0 = SecurityGroupIngress(
+            "AnotherSGRule0",
+            ToPort=443,
+            FromPort=443,
+            SourceSecurityGroupName=Ref(BaseHostSG),
+            IpProtocol="tcp",
+            GroupId=Ref(AnotherSG),
+        )
+        known = [AnotherSG, AnotherSGRule0, BaseHostLaunchConfig, BaseHostSG, BaseHostSGRule0, BaseHostSGRule1, ScalingGroup]
         config = ConfigParser(
             ProjectConfig(
                 'tests/sample-project.yaml',