Merge 63922fb into 2038504

CHANGELOG.md

@@ -1,3 +1,8 @@
+##UNRELEASED
+
+* Use a automatic resource naming to allow S3 bucket names to be auto named
+by AWS
+
 ## Version 0.5.4
 
 * Add an SSL cipher list policy pindown: implicitly (no YAML entry needed)

bootstrap_cfn/cloudformation.py

@@ -58,11 +58,28 @@ def get_stack_load_balancers(self, stack_name_or_id):
             load_balancers: Set of stack resources containing only
                 load balancers for this stack
         """
+        resource_type = 'AWS::ElasticLoadBalancing::LoadBalancer'
+        return self.get_resource_type(stack_name_or_id, resource_type)
+
+    def get_resource_type(self, stack_name_or_id, resource_type=None):
+        """
+        Collect up a set of specific stack resources
+
+        Args:
+            stack_name_or_id (string): Name or id used to identify the stack
+            resource_type(string): The resource type identifier
+
+        Returns:
+            resources: Set of stack resources containing only
+                the resource type for this stack
+        """
         # get the stack
-        load_balancers = []
+        resources = []
         stack = self.conn_cfn.describe_stacks(stack_name_or_id)
         if stack:
-            fn = lambda x: x.resource_type == 'AWS::ElasticLoadBalancing::LoadBalancer'
-            # get the load balancers group
-            load_balancers = filter(fn, stack[0].list_resources())
-        return load_balancers
+            resources = stack[0].list_resources()
+            if resource_type:
+                # get the resources
+                resources = filter(lambda x: x.resource_type == resource_type,
+                                   resources)
+        return resources

bootstrap_cfn/config.py

@@ -88,8 +88,7 @@ def process(self):
             ))
 
         if 's3' in self.data:
-            s3 = self.s3()
-            map(template.add_resource, s3)
+            self.s3(template)
 
         template = json.loads(template.to_json())
         if 'includes' in self.data:
@@ -261,32 +260,44 @@ def iam(self):
         # return json.loads(json.dumps(dict((r.title, r) for r in resources), cls=awsencode))
         return resources
 
-    def s3(self):
-        # REQUIRED FIELDS AND MAPPING
-        required_fields = {
-            'static-bucket-name': 'BucketName'
-        }
-
-        # TEST FOR REQUIRED FIELDS AND EXIT IF MISSING ANY
-        present_keys = self.data['s3'].keys()
-        for i in required_fields.keys():
-            if i not in present_keys:
-                print "\n\n[ERROR] Missing S3 fields [%s]" % i
-                sys.exit(1)
-
+    def s3(self, template):
+        """
+        Create an s3 resource configuration from the config file data.
+        This will produce Bucket and BucketPolicy resources along with
+        the bucket name as output, these are all added to the troposphere
+        template.
+
+        Args:
+            template:
+                The cloudformation template file
+        """
+        # As there are no required fields, although we may not have any
+        # subkeys we still need to be able to have a parent key 's3:' to
+        # signify that we want to create an s3 bucket. In this case we
+        # set up an empty (no options set) dictionary
+        present_keys = {}
+        if isinstance(self.data['s3'], dict):
+            present_keys = self.data['s3'].keys()
+
+        # If the static bucket name is manually set then use that,
+        # otherwise use the <stackname>-<logical-resource-name>-<random>
+        # default
         bucket = Bucket(
             "StaticBucket",
             AccessControl="BucketOwnerFullControl",
-            BucketName=self.data['s3']['static-bucket-name'],
         )
+        if 'static-bucket-name' in present_keys:
+            bucket.BucketName = self.data['s3']['static-bucket-name']
 
+        # If a policy has been manually set then use it, otherwise set
+        # a reasonable default of public 'Get' access
         if 'policy' in present_keys:
             policy = json.loads(open(self.data['s3']['policy']).read())
         else:
-            arn = 'arn:aws:s3:::%s/*' % self.data['s3']['static-bucket-name']
+            arn = Join("", ["arn:aws:s3:::", Ref(bucket), "/*"])
             policy = {
                 'Action': ['s3:GetObject'],
-                'Resource': arn,
+                "Resource": arn,
                 'Effect': 'Allow',
                 'Principal': '*'}
 
@@ -295,10 +306,16 @@ def s3(self):
             Bucket=Ref(bucket),
             PolicyDocument={"Statement": [policy]},
         )
+        # Add the bucket name to the list of cloudformation
+        # outputs
+        template.add_output(Output(
+            "StaticBucketName",
+            Description="S3 bucket name",
+            Value=Ref(bucket)
+        ))
 
-        resources = [bucket, bucket_policy]
-        # Hack until we return troposphere objects directly
-        return resources
+        # Add the resources to the troposphere template
+        map(template.add_resource, [bucket, bucket_policy])
 
     def ssl(self):
         return self.data['ssl']
@@ -465,7 +482,7 @@ def elb(self):
 
             elb_role_policies = PolicyType(
                 "Policy" + safe_name,
-                PolicyName=safe_name+"BaseHost",
+                PolicyName=safe_name + "BaseHost",
                 PolicyDocument={"Statement": [{
                     "Action": [
                         "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
@@ -478,7 +495,7 @@ def elb(self):
                             ":",
                             Ref("AWS::AccountId"),
                             ':loadbalancer/%s' % load_balancer.LoadBalancerName
-                            ])
+                        ])
                     ],
                     "Effect": "Allow"}
                 ]},
@@ -507,7 +524,7 @@ def elb(self):
                             "FromPort": 443,
                             "ToPort": 443,
                             "CidrIp": "0.0.0.0/0"
-                            },
+                        },
                         {
                             "IpProtocol": "tcp",
                             "FromPort": 80,

bootstrap_cfn/ec2.py

@@ -3,6 +3,7 @@
 from bootstrap_cfn import cloudformation
 from bootstrap_cfn import utils
 
+
 class EC2:
 
     conn_cfn = None

bootstrap_cfn/elb.py

@@ -67,7 +67,7 @@ def set_ssl_certificates(self, ssl_config, stack_name):
                         if protocol == "HTTPS":
                             logging.info("ELB::set_ssl_certificates: "
                                          "Found HTTPS protocol on '%s', "
-                                         "updating SSL certificate with '%s'" 
+                                         "updating SSL certificate with '%s'"
                                          % (load_balancer.name, cert_arn))
                             self.conn_elb.set_lb_listener_SSL_certificate(load_balancer.name,
                                                                           out_port,
@@ -83,4 +83,3 @@ def set_ssl_certificates(self, ssl_config, stack_name):
                                                  "No load balancers found in stack,")
 
         return updated_load_balancers
-

bootstrap_cfn/errors.py

@@ -4,7 +4,7 @@
 class BootstrapCfnError(Exception):
     def __init__(self, msg):
         super(BootstrapCfnError, self).__init__(msg)
-        print >> sys.stderr,  "[ERROR] {0}: {1}".format(self.__class__.__name__, msg)
+        print >> sys.stderr, "[ERROR] {0}: {1}".format(self.__class__.__name__, msg)
 
 
 class CfnConfigError(BootstrapCfnError):

bootstrap_cfn/utils.py

@@ -78,7 +78,6 @@ def tail(stack, stack_name):
     from fabric.colors import green, red, yellow
     """Show and then tail the event log"""
 
-
     def colorize(e):
         if e.endswith("_IN_PROGRESS"):
             return yellow(e)

tests/tests.py

@@ -8,7 +8,7 @@
 
 from testfixtures import compare
 
-from troposphere import Base64, FindInMap, GetAZs, GetAtt, Join, Ref, awsencode, ec2, iam, rds, s3
+from troposphere import Base64, FindInMap, GetAZs, GetAtt, Join, Ref, Template, awsencode, ec2, iam, rds, s3
 from troposphere.autoscaling import AutoScalingGroup, LaunchConfiguration, Tag
 from troposphere.ec2 import SecurityGroup, SecurityGroupIngress
 from troposphere.elasticloadbalancing import ConnectionDrainingPolicy, HealthCheck, LoadBalancer, Policy
@@ -119,12 +119,14 @@ def test_s3(self):
         bucket_ref = Ref(bucket)
 
         static_bp = s3.BucketPolicy('StaticBucketPolicy')
+        resource_value = Join("", ["arn:aws:s3:::", {"Ref": "StaticBucket"}, "/*"])
+
         static_bp.PolicyDocument = {
             'Statement': [
                 {
                     'Action': [
                         's3:GetObject'],
-                    'Resource': 'arn:aws:s3:::moj-test-dev-static/*',
+                    'Resource': resource_value,
                     'Effect': 'Allow',
                     'Principal': '*'
                 }
@@ -137,20 +139,64 @@ def test_s3(self):
                 'tests/sample-project.yaml',
                 'dev').config,
             'my-stack-name')
+
+        # Create S3 resources in template
+        template = Template()
+        config.s3(template)
+        resources = template.resources.values()
+
+        compare(self._resources_to_dict([static_bp, bucket]),
+                self._resources_to_dict(resources))
+
+    def test_s3_no_subkeys(self):
+        """
+        Test that a config with the s3: key alone will load
+        """
+        bucket = s3.Bucket('StaticBucket')
+        bucket.AccessControl = 'BucketOwnerFullControl'
+        bucket_ref = Ref(bucket)
+
+        static_bp = s3.BucketPolicy('StaticBucketPolicy')
+        resource_value = Join("", ["arn:aws:s3:::", {"Ref": "StaticBucket"}, "/*"])
+
+        static_bp.PolicyDocument = {
+            'Statement': [
+                {
+                    'Action': [
+                        's3:GetObject'],
+                    'Resource': resource_value,
+                    'Effect': 'Allow',
+                    'Principal': '*'
+                }
+            ]
+        }
+        static_bp.Bucket = bucket_ref
+
+        # Load project config and wipe out all s3 subkeys
+        base_config = ProjectConfig('tests/sample-project.yaml', 'dev')
+        base_config.config["s3"] = None
+        config = ConfigParser(base_config.config,
+                              'my-stack-name')
+        # Create S3 resources in template
+        template = Template()
+        config.s3(template)
+        resources = template.resources.values()
+
         compare(self._resources_to_dict([static_bp, bucket]),
-                self._resources_to_dict(config.s3()))
+                self._resources_to_dict(resources))
 
     def test_custom_s3_policy(self):
+        resource_value = 'arn:aws:s3:::moj-test-dev-static/*'
         expected_s3 = [
             {
                 'Action': [
                     's3:Get*',
                     's3:Put*',
                     's3:List*',
                     's3:Delete*'],
-                'Resource': 'arn:aws:s3:::moj-test-dev-static/*',
-                            'Effect': 'Allow',
-                            'Principal': {'AWS': '*'}
+                'Resource': resource_value,
+                'Effect': 'Allow',
+                'Principal': {'AWS': '*'}
             }
         ]
 
@@ -161,7 +207,13 @@ def test_custom_s3_policy(self):
             'policy': 'tests/sample-custom-s3-policy.json'}
 
         config = ConfigParser(project_config.config, 'my-stack-name')
-        s3_cfg = self._resources_to_dict(config.s3())
+
+        # Create S3 resources in template
+        template = Template()
+        config.s3(template)
+        resources = template.resources.values()
+
+        s3_cfg = self._resources_to_dict(resources)
         s3_custom_cfg = s3_cfg['StaticBucketPolicy'][
             'Properties']['PolicyDocument']['Statement']
 
@@ -467,6 +519,10 @@ def test_cf_includes(self):
             "someoutput": {
                 "Description": "For tests",
                 "Value": "BLAHBLAH"
+            },
+            "StaticBucketName": {
+                "Description": "S3 bucket name",
+                "Value": {"Ref": "StaticBucket"}
             }
         }
         config = ConfigParser(project_config.config, 'my-stack-name')
@@ -498,14 +554,14 @@ def test_process(self):
             "RolePolicies", "ScalingGroup", "StaticBucket",
             "StaticBucketPolicy", "SubnetA", "SubnetB", "SubnetC",
             "SubnetRouteTableAssociationA", "SubnetRouteTableAssociationB",
-            "SubnetRouteTableAssociationC", "VPC"
+            "SubnetRouteTableAssociationC", "VPC",
         ]
 
         resource_names = cfn_template['Resources'].keys()
         resource_names.sort()
         compare(resource_names, wanted)
 
-        wanted = ["dbhost", "dbport"]
+        wanted = ["StaticBucketName", "dbhost", "dbport"]
         output_names = cfn_template['Outputs'].keys()
         output_names.sort()
         compare(output_names, wanted)
@@ -561,7 +617,7 @@ def test_process_with_vpc_config(self):
         resource_names.sort()
         compare(resource_names, wanted)
 
-        wanted = ["dbhost", "dbport"]
+        wanted = ["StaticBucketName", "dbhost", "dbport"]
         output_names = cfn_template['Outputs'].keys()
         output_names.sort()
         compare(output_names, wanted)