Merge pull request #28 from brettswift/feature/pipeline_separate_buck…

…et_step

Enable optional S3 bucket creation

cumulus/steps/dev_tools/__init__.py

@@ -1,4 +1,4 @@
-META_PIPELINE_BUCKET_REF = 'dev_tools-bucket-Ref'
+META_PIPELINE_BUCKET_NAME = 'dev_tools-pipeline-bucket-name'
 
 # Use this to get a Ref object to gain access to the pipeline bucket
 META_PIPELINE_BUCKET_POLICY_REF = 'pipeline-bucket-access-policy-Ref'

cumulus/steps/dev_tools/code_build_action.py

@@ -19,7 +19,7 @@
 from cumulus.chain import step
 
 from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, \
-    META_PIPELINE_BUCKET_REF
+    META_PIPELINE_BUCKET_NAME
 
 
 class CodeBuildAction(step.Step):
@@ -83,7 +83,7 @@ def handle(self, chain_context):
                 Type='LINUX_CONTAINER',
                 EnvironmentVariables=[
                     # TODO: allow these to be injectable, or just the whole environment?
-                    {'Name': 'PIPELINE_BUCKET', 'Value': chain_context.metadata[META_PIPELINE_BUCKET_REF]}
+                    {'Name': 'PIPELINE_BUCKET', 'Value': chain_context.metadata[META_PIPELINE_BUCKET_NAME]}
                 ],
             )
 

cumulus/steps/dev_tools/pipeline.py

@@ -23,20 +23,23 @@ class Pipeline(step.Step):
     def __init__(self,
                  name,
                  bucket_name,
+                 create_bucket=True,
                  pipeline_policies=None,
                  bucket_policy_statements=None,
                  bucket_kms_key_arn=None,
                  ):
         """
 
+        :type create_bucket: bool if False, will not create the bucket. Will attach policies either way.
+        :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name
         :type bucket_policy_statements: [awacs.aws.Statement]
-        :type bucket: troposphere.s3.Bucket
         :type pipeline_policies: [troposphere.iam.Policy]
-        :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name
+        :type bucket_kms_key_arn: ARN used to decrypt the pipeline artifacts
         """
         step.Step.__init__(self)
         self.name = name
         self.bucket_name = bucket_name
+        self.create_bucket = create_bucket
         self.bucket_policy_statements = bucket_policy_statements
         self.pipeline_policies = pipeline_policies or []
         self.bucket_kms_key_arn = bucket_kms_key_arn
@@ -50,20 +53,22 @@ def handle(self, chain_context):
         :param chain_context:
         :return:
         """
-        # TODO: let (force?) bucket to be injected.
-        pipeline_bucket = Bucket(
-            "PipelineBucket%s" % self.name,
-            BucketName=self.bucket_name,
-            VersioningConfiguration=VersioningConfiguration(
-                Status="Enabled"
+
+        if self.create_bucket:
+            pipeline_bucket = Bucket(
+                "PipelineBucket%s" % chain_context.instance_name,
+                BucketName=self.bucket_name,
+                VersioningConfiguration=VersioningConfiguration(
+                    Status="Enabled"
+                )
             )
-        )
+            chain_context.template.add_resource(pipeline_bucket)
 
-        default_bucket_policies = self.get_default_bucket_policy_statements(pipeline_bucket)
+        default_bucket_policies = self.get_default_bucket_policy_statements(self.bucket_name)
 
         if self.bucket_policy_statements:
             bucket_access_policy = self.get_bucket_policy(
-                pipeline_bucket=pipeline_bucket,
+                pipeline_bucket=self.bucket_name,
                 bucket_policy_statements=self.bucket_policy_statements,
             )
             chain_context.template.add_resource(bucket_access_policy)
@@ -78,9 +83,7 @@ def handle(self, chain_context):
             )
         )
 
-        chain_context.template.add_resource(pipeline_bucket_access_policy)
-        # pipeline_bucket could be a string or Join object.. unit test this.
-        chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_REF] = Ref(pipeline_bucket)
+        chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_NAME] = self.bucket_name
         chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref(
             pipeline_bucket_access_policy)
 
@@ -98,12 +101,12 @@ def handle(self, chain_context):
                         Resource=[
                             troposphere.Join('', [
                                 awacs.s3.ARN(),
-                                Ref(pipeline_bucket),
+                                self.bucket_name,
                                 "/*"
                             ]),
                             troposphere.Join('', [
                                 awacs.s3.ARN(),
-                                Ref(pipeline_bucket),
+                                self.bucket_name,
                             ]),
                         ],
                     ),
@@ -173,7 +176,7 @@ def handle(self, chain_context):
             Stages=[],
             ArtifactStore=codepipeline.ArtifactStore(
                 Type="S3",
-                Location=Ref(pipeline_bucket),
+                Location=self.bucket_name,
             )
             # TODO: optionally add kms key here
         )
@@ -191,11 +194,17 @@ def handle(self, chain_context):
             Description="Code Pipeline",
             Value=Ref(generic_pipeline),
         )
+        pipeline_bucket_output = troposphere.Output(
+            "PipelineBucket",
+            Description="Name of the input artifact bucket for the pipeline",
+            Value=self.bucket_name,
+        )
 
-        chain_context.template.add_resource(pipeline_bucket)
+        chain_context.template.add_resource(pipeline_bucket_access_policy)
         chain_context.template.add_resource(pipeline_service_role)
         chain_context.template.add_resource(generic_pipeline)
         chain_context.template.add_output(pipeline_output)
+        chain_context.template.add_output(pipeline_bucket_output)
 
     def get_default_bucket_policy_statements(self, pipeline_bucket):
         bucket_policy_statements = [
@@ -208,7 +217,7 @@ def get_default_bucket_policy_statements(self, pipeline_bucket):
                 Resource=[
                     troposphere.Join('', [
                         awacs.s3.ARN(),
-                        Ref(pipeline_bucket),
+                        pipeline_bucket,
                     ]),
                 ],
             ),
@@ -236,7 +245,7 @@ def get_default_bucket_policy_statements(self, pipeline_bucket):
                 Resource=[
                     troposphere.Join('', [
                         awacs.s3.ARN(),
-                        Ref(pipeline_bucket),
+                        pipeline_bucket,
                         '/*'
                     ]),
                 ],
@@ -248,7 +257,7 @@ def get_default_bucket_policy_statements(self, pipeline_bucket):
     def get_bucket_policy(self, pipeline_bucket, bucket_policy_statements):
         policy = troposphere.s3.BucketPolicy(
             "PipelineBucketPolicy",
-            Bucket=troposphere.Ref(pipeline_bucket),
+            Bucket=pipeline_bucket,
             PolicyDocument=awacs.aws.Policy(
                 Statement=bucket_policy_statements,
             ),

cumulus/steps/storage/s3bucket.py

@@ -0,0 +1,45 @@
+from troposphere.s3 import Bucket, VersioningConfiguration
+
+from cumulus.chain import step
+
+
+class S3Bucket(step.Step):
+    def __init__(self,
+                 logical_name,
+                 bucket_name,
+                 # bucket_policy_statements=None,
+                 ):
+        """
+
+        :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name
+        :type bucket_policy_statements: [awacs.aws.Statement]
+        """
+        step.Step.__init__(self)
+        self.logical_name = logical_name
+        self.bucket_name = bucket_name
+        # TODO: this property is a vistigial one from when this was ripped out of the pipeline,
+        # however, leaving it here as it is surely useful if you want to just create a bucket
+        # with some policies.
+        # self.bucket_policy_statements = bucket_policy_statements
+
+    def handle(self, chain_context):
+        """
+        This step adds in the shell of a pipeline.
+         * s3 bucket
+         * policies for the bucket and pipeline
+         * your next step in the chain MUST be a source stage
+        :param chain_context:
+        :return:
+        """
+
+        bucket = Bucket(
+            self.logical_name,
+            BucketName=self.bucket_name,
+            VersioningConfiguration=VersioningConfiguration(
+                Status="Enabled"
+            )
+        )
+
+        chain_context.template.add_resource(bucket)
+
+        print("Added bucket: " + self.logical_name)

tests/stacker_test/blueprints/pipeline_simple.py

@@ -1,4 +1,5 @@
 import troposphere
+from cumulus.steps.storage.s3bucket import S3Bucket
 from stacker.blueprints.base import Blueprint
 import troposphere.codebuild
 
@@ -17,7 +18,6 @@ class PipelineSimple(Blueprint):
     }
 
     def create_template(self):
-
         t = self.template
         t.add_description("Acceptance Tests for cumulus pipelines")
 
@@ -34,10 +34,24 @@ def create_template(self):
             "automatedtests"
         ])
 
-        the_chain.add(pipeline.Pipeline(
+        bucket = S3Bucket(
+            logical_name="PipelineBucket",
+            bucket_name=pipeline_bucket_name,
+        )
+        # expected
+        # cumulus-acc-964705782699-automatedtests
+        # actual
+        # acc-964705782699-automatedtests
+
+        the_chain.add(bucket)
+
+        the_pipeline = pipeline.Pipeline(
             name=self.name,
             bucket_name=pipeline_bucket_name,
-        ))
+            create_bucket=False,
+        )
+
+        the_chain.add(the_pipeline)
 
         source_stage_name = "SourceStage"
         deploy_stage_name = "DeployStage"
@@ -52,7 +66,7 @@ def create_template(self):
                 action_name="MicroserviceSource",
                 output_artifact_name=service_artifact,
                 s3_bucket_name=pipeline_bucket_name,
-                s3_object_key="artifact.tar.gz"
+                s3_object_key="artifact.zip"
             )
         )
 

tests/stacker_test/conf/acceptance.env

@@ -2,3 +2,8 @@
 # http://stacker.readthedocs.io/en/latest/environments.html
 namespace: acc
 env: ac
+VpcId: vpc-894b89ef
+BaseDomain: playpen.dsl.aws.shaw.ca
+PrivateSubnets: subnet-7b8cba32,subnet-ed041b8a
+SshKeyName: stc-admin-March-2017-PLAYPEN
+ALBCertName: ${ssmstore us-west-2@/simpleweb/bswift/ALBCertName}

tests/stacker_test/run-integration.sh

@@ -1,30 +1,30 @@
 #!/usr/bin/env bash
 
+#set -x
 ACCOUNT_ID=`aws sts get-caller-identity | jq .Account | tr -d '"' `
 NAMESPACE=acc # must match the namespace in the conf file
-BUCKET="cumulus-${NAMESPACE}-${ACCOUNT_ID}-automatedtests"
 
 echo "Using account: ${ACCOUNT_ID}"
-echo "Using bucket: ${BUCKET}"
 
 set -e #Important. Script will exit appropriately if there is an error.
 
-stacker build conf/acceptance.env stacker.yaml --recreate-failed -t
+stacker build conf/acceptance.env stacker.yaml --recreate-failed -t --stacks pipelinesimple
 
-ARTIFACT_NAME='artifact.tar.gz'
+BUCKET_NAME=$(stacker info conf/acceptance.env stacker.yaml 2>&1 | grep PipelineBucket: | cut -f 3 -d " ")
+PIPELINE_NAME=$(stacker info conf/acceptance.env stacker.yaml 2>&1 | grep PipelineName | cut -f 3 -d " ")
+
+ARTIFACT_NAME='artifact.zip'
 TEMP_DIR='ac_build'
 
 pushd ../../ # move to main folder
 mkdir -p ${TEMP_DIR}
-zip -r ${TEMP_DIR}/${ARTIFACT_NAME} ./ -x *.git* *./${TEMP_DIR}* *.eggs* *.idea* *.tox*
+zip -r ${TEMP_DIR}/${ARTIFACT_NAME} ./ -x *.git* *./${TEMP_DIR}* *.eggs* *.egg-* *.idea* *.tox* *tests* *docs*
 
-aws s3 cp ./${TEMP_DIR}/${ARTIFACT_NAME} s3://${BUCKET}/${ARTIFACT_NAME}
+aws s3 cp ./${TEMP_DIR}/${ARTIFACT_NAME} s3://${BUCKET_NAME}/${ARTIFACT_NAME}
 
 rm -rf ${TEMP_DIR}
 popd # return to test folder
 
-PIPELINE_NAME=$(stacker info conf/acceptance.env stacker.yaml 2>&1 | grep PipelineLogicalName | cut -f 3 -d " ")
-
 echo "Pipeline deployment started for pipeline: ${PIPELINE_NAME}"
 
 # get shasum from expected and actual output. When they match we are at approval state
@@ -60,8 +60,8 @@ done
 
 SHOULD_DESTROY=false
 if $SHOULD_DESTROY; then
-    aws s3 rm s3://${BUCKET} --recursive
-    python delete_bucket_versions.py ${BUCKET}
+    aws s3 rm s3://${BUCKET_NAME} --recursive
+    python delete_bucket_versions.py ${BUCKET_NAME}
 
     stacker destroy conf/acceptance.env stacker.yaml --force -t
 fi

tests/unit/steps/test_pipeline.py

@@ -8,7 +8,7 @@
 import unittest
 
 import troposphere
-from troposphere import codepipeline, codebuild
+from troposphere import codepipeline, codebuild, s3
 
 from cumulus.chain import chaincontext
 from cumulus.steps.dev_tools import pipeline, code_build_action
@@ -56,6 +56,31 @@ def test_pipeline_has_two_stages(self):
 
         pipelines = TemplateQuery.get_resource_by_type(t, codepipeline.Pipeline)
         self.assertTrue(len(pipelines), 1)
+
+    def test_pipeline_creates_default_bucket(self):
+
+        sut = pipeline.Pipeline(
+            name='test',
+            bucket_name='testbucket',
+        )
+        sut.handle(self.context)
+        t = self.context.template
+
+        the_bucket = TemplateQuery.get_resource_by_type(t, s3.Bucket)
+        self.assertTrue(len(the_bucket), 1)
+
+    def test_pipeline_uses_non_default_bucket(self):
+        sut = pipeline.Pipeline(
+            name='test',
+            bucket_name='testbucket',
+            create_bucket=False,
+        )
+        sut.handle(self.context)
+        t = self.context.template
+
+        the_bucket = TemplateQuery.get_resource_by_type(t, s3.Bucket)
+        self.assertEqual(len(the_bucket), 0)
+
     #
     # def test_codebuild_should_add_stage(self):
     #