Skip to content

Commit

Permalink
Merge pull request #23 from brettswift/feature/modifications_for_cros…
Browse files Browse the repository at this point in the history 
…s_account_pipeline

Support for injectable cloudformation roles
  • Loading branch information
@brettswift
brettswift committed Oct 15, 2018
2 parents dc44bce + 7c19a32 commit 74b26c3
Show file tree
Hide file tree
Showing 4 changed files with 271 additions and 97 deletions.
96 changes: 63 additions & 33 deletions cumulus/steps/dev_tools/cloud_formation_action.py
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import awacs
import awacs.aws
import awacs.sts
from troposphere import iam, codepipeline, GetAtt

import cumulus.policies
import cumulus.policies.cloudformation
import cumulus.types.codebuild.buildaction

from troposphere import iam, codepipeline, GetAtt
import cumulus.util.template_query
from cumulus.chain import step
from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF


class CloudFormationAction(step.Step):

OUTPUT_FILE_NAME = 'StackOutputs.json'

def __init__(self,
Expand All @@ -21,8 +22,12 @@ def __init__(self,
stage_name_to_add,
stack_name,
action_mode,
output_artifact_name=None):
output_artifact_name=None,
cfn_action_role_arn=None,
cfn_action_config_role_arn=None,
):
"""
:type cfn_action_config_role_arn: [troposphere.iam.Policy]
:type action_name: basestring Displayed on the console
:type input_artifact_names: [basestring] List of input artifacts
:type input_template_path: basestring Full path to cloudformation template (ex. ArtifactName::templatefolder/template.json)
Expand All @@ -40,35 +45,21 @@ def __init__(self,
self.stack_name = stack_name
self.action_mode = action_mode
self.output_artifact_name = output_artifact_name
self.cfn_action_role_arn = cfn_action_role_arn
self.cfn_action_config_role_arn = cfn_action_config_role_arn

def handle(self, chain_context):

print("Adding action %sstage" % self.action_name)

policy_name = "CloudFormationPolicy%stage" % chain_context.instance_name
role_name = "CloudFormationRole%stage" % self.action_name

cloud_formation_role = iam.Role(
role_name,
Path="/",
AssumeRolePolicyDocument=awacs.aws.Policy(
Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.sts.AssumeRole],
Principal=awacs.aws.Principal(
'Service',
["cloudformation.amazonaws.com"]
)
)]
),
Policies=[
cumulus.policies.cloudformation.get_policy_cloudformation_general_access(policy_name)
],
ManagedPolicyArns=[
chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF]
]
)
# if supplied, use the role injected in, otherwise, build one.
if self.cfn_action_config_role_arn:
cfn_configuration_role_arn = self.cfn_action_config_role_arn
else:
cfn_configuration_role = self.get_cfn_role(
chain_context=chain_context,
)
cfn_configuration_role_arn = GetAtt(cfn_configuration_role, 'Arn')
chain_context.template.add_resource(cfn_configuration_role)

input_artifacts = []
for artifact_name in self.input_artifact_names:
Expand All @@ -81,7 +72,8 @@ def handle(self, chain_context):
InputArtifacts=input_artifacts,
Configuration={
'ActionMode': self.action_mode.value,
'RoleArn': GetAtt(cloud_formation_role, 'Arn'),
# this role needs to be the cfn role above, and it should add the tools account policy
'RoleArn': cfn_configuration_role_arn,
'StackName': self.stack_name,
'Capabilities': 'CAPABILITY_NAMED_IAM',
'TemplateConfiguration': self.input_template_configuration,
Expand All @@ -91,7 +83,7 @@ def handle(self, chain_context):
)

# Add optional configuration
if (self.output_artifact_name):
if self.output_artifact_name:
output_artifact = codepipeline.OutputArtifacts(
Name=self.output_artifact_name
)
Expand All @@ -100,14 +92,52 @@ def handle(self, chain_context):
]
cloud_formation_action.Configuration['OutputFileName'] = CloudFormationAction.OUTPUT_FILE_NAME

chain_context.template.add_resource(cloud_formation_role)
if self.cfn_action_role_arn:
cloud_formation_action.RoleArn = self.cfn_action_role_arn

stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name(
template=chain_context.template,
stage_name=self.stage_name_to_add
stage_name=self.stage_name_to_add,
)

# TODO accept a parallel action to the previous action, and don't +1 here.
next_run_order = len(stage.Actions) + 1
cloud_formation_action.RunOrder = next_run_order
stage.Actions.append(cloud_formation_action)

def get_cfn_role(self, chain_context, step_policies=None):
"""
Default role for cloudformation with access to the S3 bucket and cloudformation assumerole.
:param chain_context: chaincontext.ChainContext
:type step_policies: [troposphere.iam.Policy]
"""
policy_name = "CloudFormationPolicy%stage" % chain_context.instance_name
role_name = "CloudFormationRole%stage" % self.action_name

all_policies = [
cumulus.policies.cloudformation.get_policy_cloudformation_general_access(policy_name)
]

if step_policies:
all_policies += step_policies

cloud_formation_role = iam.Role(
role_name,
Path="/",
AssumeRolePolicyDocument=awacs.aws.Policy(
Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.sts.AssumeRole],
Principal=awacs.aws.Principal(
'Service',
["cloudformation.amazonaws.com"]
)
)]
),
Policies=all_policies,
ManagedPolicyArns=[
chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF]
]
)
return cloud_formation_role
163 changes: 102 additions & 61 deletions cumulus/steps/dev_tools/pipeline.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

import awacs
import troposphere

Expand All @@ -17,19 +18,28 @@
from troposphere import codepipeline, Ref, iam
from troposphere.s3 import Bucket, VersioningConfiguration

# SOURCE_STAGE_OUTPUT_NAME = 'SourceStageOutput'


class Pipeline(step.Step):

def __init__(self, name, bucket_name):
def __init__(self,
name,
bucket_name,
pipeline_policies=None,
bucket_policy_statements=None,
bucket_kms_key_arn=None,
):
"""
:type bucket_policy_statements: [awacs.aws.Statement]
:type bucket: troposphere.s3.Bucket
:type pipeline_policies: [troposphere.iam.Policy]
:type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name
"""
step.Step.__init__(self)
self.name = name
self.bucket_name = bucket_name
self.bucket_policy_statements = bucket_policy_statements
self.pipeline_policies = pipeline_policies or []
self.bucket_kms_key_arn = bucket_kms_key_arn

def handle(self, chain_context):
"""
Expand All @@ -49,64 +59,32 @@ def handle(self, chain_context):
)
)

default_bucket_policies = self.get_default_bucket_policy_statements(pipeline_bucket)

if self.bucket_policy_statements:
bucket_access_policy = self.get_bucket_policy(
pipeline_bucket=pipeline_bucket,
bucket_policy_statements=self.bucket_policy_statements,
)
chain_context.template.add_resource(bucket_access_policy)

pipeline_bucket_access_policy = iam.ManagedPolicy(
"PipelineBucketAccessPolicy",
Path='/managed/',
PolicyDocument=awacs.aws.PolicyDocument(
Version="2012-10-17",
Id="bucket-access-policy%s" % chain_context.instance_name,
Statement=[
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.s3.ListBucket,
awacs.s3.GetBucketVersioning,
],
Resource=[
troposphere.Join('', [
awacs.s3.ARN(),
Ref(pipeline_bucket),
]),
],
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.s3.HeadBucket,
],
Resource=[
'*'
]
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.s3.GetObject,
awacs.s3.GetObjectVersion,
awacs.s3.PutObject,
awacs.s3.ListObjects,
awacs.s3.ListBucketMultipartUploads,
awacs.s3.AbortMultipartUpload,
awacs.s3.ListMultipartUploadParts,
awacs.aws.Action("s3", "Get*"),
],
Resource=[
troposphere.Join('', [
awacs.s3.ARN(),
Ref(pipeline_bucket),
'/*'
]),
],
)
]
Statement=default_bucket_policies
)
)

chain_context.template.add_resource(pipeline_bucket_access_policy)
# pipeline_bucket could be a string or Join object.. unit test this.
chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_REF] = Ref(pipeline_bucket)
chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref(pipeline_bucket_access_policy)
chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref(
pipeline_bucket_access_policy)

# TODO: this can be cleaned up by using a policytype and passing in the pipeline role it should add itself to.
pipeline_policy = iam.Policy(
PolicyName="%sPolicy" % self.name,
PolicyDocument=awacs.aws.PolicyDocument(
Expand All @@ -123,18 +101,17 @@ def handle(self, chain_context):
Ref(pipeline_bucket),
"/*"
]),
],
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.aws.Action("s3", "*")],
Resource=[
troposphere.Join('', [
awacs.s3.ARN(),
Ref(pipeline_bucket),
]),
],
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[awacs.aws.Action("kms", "*")],
Resource=['*'],
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
Expand Down Expand Up @@ -176,7 +153,6 @@ def handle(self, chain_context):
pipeline_service_role = iam.Role(
"PipelineServiceRole",
Path="/",
RoleName="PipelineRole%s" % chain_context.instance_name,
AssumeRolePolicyDocument=awacs.aws.Policy(
Statement=[
awacs.aws.Statement(
Expand All @@ -188,21 +164,28 @@ def handle(self, chain_context):
)
)]
),
Policies=[
pipeline_policy
]
Policies=[pipeline_policy] + self.pipeline_policies
)

generic_pipeline = codepipeline.Pipeline(
"Pipeline",
# Name=chain_context.instance_name,
RoleArn=troposphere.GetAtt(pipeline_service_role, "Arn"),
Stages=[],
ArtifactStore=codepipeline.ArtifactStore(
Type="S3",
Location=Ref(pipeline_bucket)
Location=Ref(pipeline_bucket),
)
# TODO: optionally add kms key here
)

if self.bucket_kms_key_arn:
encryption_config = codepipeline.EncryptionKey(
"ArtifactBucketKmsKey",
Id=self.bucket_kms_key_arn,
Type='KMS',
)
generic_pipeline.ArtifactStore.EncryptionKey = encryption_config

pipeline_output = troposphere.Output(
"PipelineName",
Description="Code Pipeline",
Expand All @@ -213,3 +196,61 @@ def handle(self, chain_context):
chain_context.template.add_resource(pipeline_service_role)
chain_context.template.add_resource(generic_pipeline)
chain_context.template.add_output(pipeline_output)

def get_default_bucket_policy_statements(self, pipeline_bucket):
bucket_policy_statements = [
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.s3.ListBucket,
awacs.s3.GetBucketVersioning,
],
Resource=[
troposphere.Join('', [
awacs.s3.ARN(),
Ref(pipeline_bucket),
]),
],
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.s3.HeadBucket,
],
Resource=[
'*'
]
),
awacs.aws.Statement(
Effect=awacs.aws.Allow,
Action=[
awacs.s3.GetObject,
awacs.s3.GetObjectVersion,
awacs.s3.PutObject,
awacs.s3.ListObjects,
awacs.s3.ListBucketMultipartUploads,
awacs.s3.AbortMultipartUpload,
awacs.s3.ListMultipartUploadParts,
awacs.aws.Action("s3", "Get*"),
],
Resource=[
troposphere.Join('', [
awacs.s3.ARN(),
Ref(pipeline_bucket),
'/*'
]),
],
)
]

return bucket_policy_statements

def get_bucket_policy(self, pipeline_bucket, bucket_policy_statements):
policy = troposphere.s3.BucketPolicy(
"PipelineBucketPolicy",
Bucket=troposphere.Ref(pipeline_bucket),
PolicyDocument=awacs.aws.Policy(
Statement=bucket_policy_statements,
),
)
return policy

0 comments on commit 74b26c3

Please sign in to comment.