Merge pull request #31 from brettswift/feature/cross_account_codebuil…

…d_support

Support for injectable cloudformation roles

cumulus/policies/codebuild.py

@@ -31,6 +31,7 @@ def get_policy_code_build_general_access(policy_name):
                         awacs.aws.Action("apigateway", "*"),
                         awacs.aws.Action("cloudwatch", "*"),
                         awacs.aws.Action("cloudfront", "*"),
+                        awacs.aws.Action("codepipeline", "*"),
                         awacs.aws.Action("rds", "*"),
                         awacs.aws.Action("dynamodb", "*"),
                         awacs.aws.Action("lambda", "*"),

cumulus/steps/dev_tools/cloud_formation_action.py

@@ -25,6 +25,7 @@ def __init__(self,
                  output_artifact_name=None,
                  cfn_action_role_arn=None,
                  cfn_action_config_role_arn=None,
+                 cfn_param_overrides=None,
                  ):
         """
         :type cfn_action_config_role_arn: [troposphere.iam.Policy]
@@ -37,6 +38,7 @@ def __init__(self,
         :type action_mode: cumulus.types.cloudformation.action_mode.ActionMode The actual CloudFormation action to execute
         """
         step.Step.__init__(self)
+        self.cfn_param_overrides = cfn_param_overrides
         self.action_name = action_name
         self.input_artifact_names = input_artifact_names
         self.input_template_path = input_template_path
@@ -95,6 +97,9 @@ def handle(self, chain_context):
         if self.cfn_action_role_arn:
             cloud_formation_action.RoleArn = self.cfn_action_role_arn
 
+        if self.cfn_param_overrides:
+            cloud_formation_action.Configuration['ParameterOverrides'] = self.cfn_param_overrides
+
         stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name(
             template=chain_context.template,
             stage_name=self.stage_name_to_add,

cumulus/steps/dev_tools/code_build_action.py

@@ -1,23 +1,19 @@
 import awacs
-import troposphere
-
 import awacs.aws
+import awacs.ec2
+import awacs.iam
 import awacs.logs
 import awacs.s3
-import awacs.iam
-import awacs.ec2
 import awacs.sts
+import troposphere
+from troposphere import iam, \
+    codebuild, codepipeline, Ref, ec2
 
 import cumulus.policies
 import cumulus.policies.codebuild
 import cumulus.types.codebuild.buildaction
 import cumulus.util.template_query
-
-from troposphere import iam,\
-    codebuild, codepipeline, Ref, ec2
-
 from cumulus.chain import step
-
 from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, \
     META_PIPELINE_BUCKET_NAME
 
@@ -26,11 +22,13 @@ class CodeBuildAction(step.Step):
 
     def __init__(self,
                  action_name,
-                 input_artifact_name,
                  stage_name_to_add,
+                 input_artifact_name,
                  environment=None,
                  vpc_config=None,
-                 buildspec='buildspec.yml'):
+                 buildspec='buildspec.yml',
+                 role_arn=None,
+                 ):
         """
         :type buildspec: basestring path to buildspec.yml or text containing the buildspec.
         :type input_artifact_name: basestring The artifact name in the pipeline. Must contain a buildspec.yml
@@ -39,6 +37,7 @@ def __init__(self,
         :type vpc_config.Vpc_Config: Only required if the codebuild step requires access to the VPC
         """
         step.Step.__init__(self)
+        self.role_arn = role_arn
         self.buildspec = buildspec
         self.environment = environment
         self.input_artifact_name = input_artifact_name
@@ -48,34 +47,22 @@ def __init__(self,
 
     def handle(self, chain_context):
 
+        self.validate(chain_context)
+
         print("Adding action %s Stage." % self.action_name)
         full_action_name = "%s%s" % (self.stage_name_to_add, self.action_name)
 
         policy_name = "%sCodeBuildPolicy" % chain_context.instance_name
         role_name = "CodeBuildRole%s" % full_action_name
 
-        codebuild_role = iam.Role(
-            role_name,
-            Path="/",
-            AssumeRolePolicyDocument=awacs.aws.Policy(
-                Statement=[
-                    awacs.aws.Statement(
-                        Effect=awacs.aws.Allow,
-                        Action=[awacs.sts.AssumeRole],
-                        Principal=awacs.aws.Principal(
-                            'Service',
-                            "codebuild.amazonaws.com"
-                        )
-                    )]
-            ),
-            Policies=[
-                cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name)
-            ],
-            ManagedPolicyArns=[
-                chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF]
-            ]
+        codebuild_role = self.get_default_code_build_role(
+            chain_context=chain_context,
+            policy_name=policy_name,
+            role_name=role_name,
         )
 
+        codebuild_role_arn = self.role_arn if self.role_arn else troposphere.GetAtt(codebuild_role, 'Arn')
+
         if not self.environment:
             self.environment = codebuild.Environment(
                 ComputeType='BUILD_GENERAL1_SMALL',
@@ -89,7 +76,7 @@ def handle(self, chain_context):
 
         project = self.create_project(
             chain_context=chain_context,
-            codebuild_role=codebuild_role,
+            codebuild_role_arn=codebuild_role_arn,
             codebuild_environment=self.environment,
             name=full_action_name,
         )
@@ -117,7 +104,31 @@ def handle(self, chain_context):
         code_build_action.RunOrder = next_run_order
         stage.Actions.append(code_build_action)
 
-    def create_project(self, chain_context, codebuild_role, codebuild_environment, name):
+    def get_default_code_build_role(self, chain_context, policy_name, role_name):
+        codebuild_role = iam.Role(
+            role_name,
+            Path="/",
+            AssumeRolePolicyDocument=awacs.aws.Policy(
+                Statement=[
+                    awacs.aws.Statement(
+                        Effect=awacs.aws.Allow,
+                        Action=[awacs.sts.AssumeRole],
+                        Principal=awacs.aws.Principal(
+                            'Service',
+                            "codebuild.amazonaws.com"
+                        )
+                    )]
+            ),
+            Policies=[
+                cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name)
+            ],
+            ManagedPolicyArns=[
+                chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF]
+            ]
+        )
+        return codebuild_role
+
+    def create_project(self, chain_context, codebuild_role_arn, codebuild_environment, name):
 
         artifacts = codebuild.Artifacts(Type='CODEPIPELINE')
 
@@ -152,11 +163,10 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n
 
         project = codebuild.Project(
             project_name,
-            DependsOn=codebuild_role,
             Artifacts=artifacts,
             Environment=codebuild_environment,
             Name="%s-%s" % (chain_context.instance_name, project_name),
-            ServiceRole=troposphere.GetAtt(codebuild_role, 'Arn'),
+            ServiceRole=codebuild_role_arn,
             Source=codebuild.Source(
                 "Deploy",
                 Type='CODEPIPELINE',
@@ -166,3 +176,8 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n
         )
 
         return project
+
+    def validate(self, chain_context):
+        if META_PIPELINE_BUCKET_POLICY_REF not in chain_context.metadata:
+            raise AssertionError("Could not find expected 'META_PIPELINE_BUCKET_POLICY_REF' in metadata. "
+                                 "Maybe you added the code build step to the chain before the pipeline step?")

cumulus/steps/dev_tools/lambda_action.py

@@ -11,7 +11,7 @@
 import cumulus.policies
 import cumulus.policies.codebuild
 import cumulus.types.codebuild.buildaction
-import cumulus.util.tropo
+import cumulus.util.template_query
 from cumulus.chain import step
 from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF
 
@@ -74,14 +74,15 @@ def handle(self, chain_context):
                 codepipeline.InputArtifacts(Name=self.input_artifact_name)
             ],
             Configuration={
-                'FunctionName': self.function_name
+                'FunctionName': self.function_name,
+                'UserParameters': self.user_parameters,
             },
             RunOrder="1"
         )
 
         chain_context.template.add_resource(lambda_role)
 
-        stage = cumulus.util.tropo.TemplateQuery.get_pipeline_stage_by_name(
+        stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name(
             template=chain_context.template,
             stage_name=self.stage_name_to_add,
         )

cumulus/steps/dev_tools/pipeline.py

@@ -1,35 +1,36 @@
 
 import awacs
-import troposphere
-
-import awacs.iam
 import awacs.aws
-import awacs.sts
-import awacs.s3
-import awacs.logs
+import awacs.awslambda
+import awacs.codecommit
 import awacs.ec2
 import awacs.iam
-import awacs.codecommit
-import awacs.awslambda
-
-from cumulus.chain import step
-import cumulus.steps.dev_tools
-
+import awacs.logs
+import awacs.s3
+import awacs.sts
+import awacs.kms
+import troposphere
 from troposphere import codepipeline, Ref, iam
 from troposphere.s3 import Bucket, VersioningConfiguration
 
+import cumulus.steps.dev_tools
+from cumulus.chain import step
+
 
 class Pipeline(step.Step):
     def __init__(self,
                  name,
                  bucket_name,
+                 pipeline_service_role_arn=None,
                  create_bucket=True,
                  pipeline_policies=None,
                  bucket_policy_statements=None,
                  bucket_kms_key_arn=None,
                  ):
         """
 
+        :type pipeline_service_role_arn: basestring Override the pipeline service role. If you pass this
+                                                     the pipeline_policies is not used.
         :type create_bucket: bool if False, will not create the bucket. Will attach policies either way.
         :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name
         :type bucket_policy_statements: [awacs.aws.Statement]
@@ -40,6 +41,7 @@ def __init__(self,
         self.name = name
         self.bucket_name = bucket_name
         self.create_bucket = create_bucket
+        self.pipeline_service_role_arn = pipeline_service_role_arn
         self.bucket_policy_statements = bucket_policy_statements
         self.pipeline_policies = pipeline_policies or []
         self.bucket_kms_key_arn = bucket_kms_key_arn
@@ -53,10 +55,9 @@ def handle(self, chain_context):
         :param chain_context:
         :return:
         """
-
         if self.create_bucket:
             pipeline_bucket = Bucket(
-                "PipelineBucket%s" % chain_context.instance_name,
+                "PipelineBucket%s" % self.name,
                 BucketName=self.bucket_name,
                 VersioningConfiguration=VersioningConfiguration(
                     Status="Enabled"
@@ -87,6 +88,47 @@ def handle(self, chain_context):
         chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref(
             pipeline_bucket_access_policy)
 
+        default_pipeline_role = self.get_default_pipeline_role()
+        pipeline_service_role_arn = self.pipeline_service_role_arn or troposphere.GetAtt(default_pipeline_role, "Arn")
+
+        generic_pipeline = codepipeline.Pipeline(
+            "Pipeline",
+            RoleArn=pipeline_service_role_arn,
+            Stages=[],
+            ArtifactStore=codepipeline.ArtifactStore(
+                Type="S3",
+                Location=self.bucket_name,
+            )
+        )
+
+        if self.bucket_kms_key_arn:
+            encryption_config = codepipeline.EncryptionKey(
+                "ArtifactBucketKmsKey",
+                Id=self.bucket_kms_key_arn,
+                Type='KMS',
+            )
+            generic_pipeline.ArtifactStore.EncryptionKey = encryption_config
+
+        pipeline_output = troposphere.Output(
+            "PipelineName",
+            Description="Code Pipeline",
+            Value=Ref(generic_pipeline),
+        )
+        pipeline_bucket_output = troposphere.Output(
+            "PipelineBucket",
+            Description="Name of the input artifact bucket for the pipeline",
+            Value=self.bucket_name,
+        )
+
+        if not self.pipeline_service_role_arn:
+            chain_context.template.add_resource(default_pipeline_role)
+
+        chain_context.template.add_resource(pipeline_bucket_access_policy)
+        chain_context.template.add_resource(generic_pipeline)
+        chain_context.template.add_output(pipeline_output)
+        chain_context.template.add_output(pipeline_bucket_output)
+
+    def get_default_pipeline_role(self):
         # TODO: this can be cleaned up by using a policytype and passing in the pipeline role it should add itself to.
         pipeline_policy = iam.Policy(
             PolicyName="%sPolicy" % self.name,
@@ -148,7 +190,7 @@ def handle(self, chain_context):
                             awacs.aws.Action("lambda", "*")
                         ],
                         Resource=["*"]
-                    )
+                    ),
                 ],
             )
         )
@@ -169,42 +211,7 @@ def handle(self, chain_context):
             ),
             Policies=[pipeline_policy] + self.pipeline_policies
         )
-
-        generic_pipeline = codepipeline.Pipeline(
-            "Pipeline",
-            RoleArn=troposphere.GetAtt(pipeline_service_role, "Arn"),
-            Stages=[],
-            ArtifactStore=codepipeline.ArtifactStore(
-                Type="S3",
-                Location=self.bucket_name,
-            )
-            # TODO: optionally add kms key here
-        )
-
-        if self.bucket_kms_key_arn:
-            encryption_config = codepipeline.EncryptionKey(
-                "ArtifactBucketKmsKey",
-                Id=self.bucket_kms_key_arn,
-                Type='KMS',
-            )
-            generic_pipeline.ArtifactStore.EncryptionKey = encryption_config
-
-        pipeline_output = troposphere.Output(
-            "PipelineName",
-            Description="Code Pipeline",
-            Value=Ref(generic_pipeline),
-        )
-        pipeline_bucket_output = troposphere.Output(
-            "PipelineBucket",
-            Description="Name of the input artifact bucket for the pipeline",
-            Value=self.bucket_name,
-        )
-
-        chain_context.template.add_resource(pipeline_bucket_access_policy)
-        chain_context.template.add_resource(pipeline_service_role)
-        chain_context.template.add_resource(generic_pipeline)
-        chain_context.template.add_output(pipeline_output)
-        chain_context.template.add_output(pipeline_bucket_output)
+        return pipeline_service_role
 
     def get_default_bucket_policy_statements(self, pipeline_bucket):
         bucket_policy_statements = [