From c43af84d0051d540b1c4f671e2f576855bf7089f Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Tue, 4 Apr 2023 14:04:32 -0700 Subject: [PATCH 1/8] beta testing fixes --- API.md | 262 ++++++++++++++++++++++++++ src/base/stack.ts | 151 ++++++++------- src/config/configurator.ts | 15 ++ src/pipelines/stage.ts | 2 +- src/stages/databrew-transform.ts | 50 +++-- src/stages/sqs-lambda.ts | 6 +- test/appflow-ingestion-stage.test.ts | 2 +- test/base-stack.test.ts | 10 +- test/config.test.ts | 9 + test/databrew-transform-stage.test.ts | 37 +++- test/kinesis-s3-stage.test.ts | 2 +- test/sqs-lambda-stage.test.ts | 2 + 12 files changed, 449 insertions(+), 99 deletions(-) diff --git a/API.md b/API.md index d1ccf795..f7c068ac 100644 --- a/API.md +++ b/API.md @@ -878,6 +878,7 @@ Convert an object, potentially containing tokens, to a JSON string. | isConstruct | Checks if `x` is a construct. | | isStack | Return whether the given object is a Stack. | | of | Looks up the first stack scope in which `construct` is defined. | +| createDefaultPermissionsBoundary | *No description.* | --- @@ -937,6 +938,32 @@ The construct to start the search from. --- +##### `createDefaultPermissionsBoundary` + +```typescript +import { BaseStack } from 'aws-ddk-core' + +BaseStack.createDefaultPermissionsBoundary(scope: Construct, id: string, props: PermissionsBoundaryProps) +``` + +###### `scope`Required + +- *Type:* constructs.Construct + +--- + +###### `id`Required + +- *Type:* string + +--- + +###### `props`Required + +- *Type:* PermissionsBoundaryProps + +--- + #### Properties | **Name** | **Type** | **Description** | @@ -1869,6 +1896,7 @@ public synth(): CICDPipelineStack | isConstruct | Checks if `x` is a construct. | | isStack | Return whether the given object is a Stack. | | of | Looks up the first stack scope in which `construct` is defined. | +| createDefaultPermissionsBoundary | *No description.* | --- @@ -1928,6 +1956,32 @@ The construct to start the search from. --- +##### `createDefaultPermissionsBoundary` + +```typescript +import { CICDPipelineStack } from 'aws-ddk-core' + +CICDPipelineStack.createDefaultPermissionsBoundary(scope: Construct, id: string, props: PermissionsBoundaryProps) +``` + +###### `scope`Required + +- *Type:* constructs.Construct + +--- + +###### `id`Required + +- *Type:* string + +--- + +###### `props`Required + +- *Type:* PermissionsBoundaryProps + +--- + #### Properties | **Name** | **Type** | **Description** | @@ -6430,12 +6484,26 @@ const dataBrewTransformStageProps: DataBrewTransformStageProps = { ... } | stateMachineInput | {[ key: string ]: any} | *No description.* | | stateMachineName | string | *No description.* | | createJob | boolean | *No description.* | +| databaseOutputs | aws-cdk-lib.aws_databrew.CfnJob.DatabaseOutputProperty[] | *No description.* | +| dataCatalogOutputs | aws-cdk-lib.aws_databrew.CfnJob.DataCatalogOutputProperty[] | *No description.* | | datasetName | string | *No description.* | +| encryptionKeyArn | string | *No description.* | +| encryptionMode | string | *No description.* | | jobName | string | *No description.* | | jobRoleArn | string | *No description.* | +| jobSample | aws-cdk-lib.aws_databrew.CfnJob.JobSampleProperty | *No description.* | | jobType | string | *No description.* | +| logSubscription | string | *No description.* | +| maxCapacity | number | *No description.* | +| maxRetries | number | *No description.* | +| outputLocation | aws-cdk-lib.aws_databrew.CfnJob.OutputLocationProperty | *No description.* | | outputs | aws-cdk-lib.aws_databrew.CfnJob.OutputProperty[] | *No description.* | +| profileConfiguration | aws-cdk-lib.aws_databrew.CfnJob.ProfileConfigurationProperty | *No description.* | +| projectName | string | *No description.* | | recipe | aws-cdk-lib.aws_databrew.CfnJob.RecipeProperty | *No description.* | +| tags | aws-cdk-lib.CfnTag[] | *No description.* | +| timeout | number | *No description.* | +| validationConfigurations | aws-cdk-lib.aws_databrew.CfnJob.ValidationConfigurationProperty[] | *No description.* | --- @@ -6529,6 +6597,26 @@ public readonly createJob: boolean; --- +##### `databaseOutputs`Optional + +```typescript +public readonly databaseOutputs: DatabaseOutputProperty[]; +``` + +- *Type:* aws-cdk-lib.aws_databrew.CfnJob.DatabaseOutputProperty[] + +--- + +##### `dataCatalogOutputs`Optional + +```typescript +public readonly dataCatalogOutputs: DataCatalogOutputProperty[]; +``` + +- *Type:* aws-cdk-lib.aws_databrew.CfnJob.DataCatalogOutputProperty[] + +--- + ##### `datasetName`Optional ```typescript @@ -6539,6 +6627,26 @@ public readonly datasetName: string; --- +##### `encryptionKeyArn`Optional + +```typescript +public readonly encryptionKeyArn: string; +``` + +- *Type:* string + +--- + +##### `encryptionMode`Optional + +```typescript +public readonly encryptionMode: string; +``` + +- *Type:* string + +--- + ##### `jobName`Optional ```typescript @@ -6559,6 +6667,16 @@ public readonly jobRoleArn: string; --- +##### `jobSample`Optional + +```typescript +public readonly jobSample: JobSampleProperty; +``` + +- *Type:* aws-cdk-lib.aws_databrew.CfnJob.JobSampleProperty + +--- + ##### `jobType`Optional ```typescript @@ -6569,6 +6687,46 @@ public readonly jobType: string; --- +##### `logSubscription`Optional + +```typescript +public readonly logSubscription: string; +``` + +- *Type:* string + +--- + +##### `maxCapacity`Optional + +```typescript +public readonly maxCapacity: number; +``` + +- *Type:* number + +--- + +##### `maxRetries`Optional + +```typescript +public readonly maxRetries: number; +``` + +- *Type:* number + +--- + +##### `outputLocation`Optional + +```typescript +public readonly outputLocation: OutputLocationProperty; +``` + +- *Type:* aws-cdk-lib.aws_databrew.CfnJob.OutputLocationProperty + +--- + ##### `outputs`Optional ```typescript @@ -6579,6 +6737,26 @@ public readonly outputs: OutputProperty[]; --- +##### `profileConfiguration`Optional + +```typescript +public readonly profileConfiguration: ProfileConfigurationProperty; +``` + +- *Type:* aws-cdk-lib.aws_databrew.CfnJob.ProfileConfigurationProperty + +--- + +##### `projectName`Optional + +```typescript +public readonly projectName: string; +``` + +- *Type:* string + +--- + ##### `recipe`Optional ```typescript @@ -6589,6 +6767,36 @@ public readonly recipe: RecipeProperty; --- +##### `tags`Optional + +```typescript +public readonly tags: CfnTag[]; +``` + +- *Type:* aws-cdk-lib.CfnTag[] + +--- + +##### `timeout`Optional + +```typescript +public readonly timeout: number; +``` + +- *Type:* number + +--- + +##### `validationConfigurations`Optional + +```typescript +public readonly validationConfigurations: ValidationConfigurationProperty[]; +``` + +- *Type:* aws-cdk-lib.aws_databrew.CfnJob.ValidationConfigurationProperty[] + +--- + ### DataPipelineProps #### Initializer @@ -7021,6 +7229,45 @@ public readonly rolePolicyStatements: PolicyStatement[]; --- +### GetTagsProps + +#### Initializer + +```typescript +import { GetTagsProps } from 'aws-ddk-core' + +const getTagsProps: GetTagsProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| configPath | string | *No description.* | +| environmentId | string | *No description.* | + +--- + +##### `configPath`Required + +```typescript +public readonly configPath: string; +``` + +- *Type:* string + +--- + +##### `environmentId`Optional + +```typescript +public readonly environmentId: string; +``` + +- *Type:* string + +--- + ### GlueTransformStageProps #### Initializer @@ -9137,6 +9384,7 @@ public tagConstruct(scope: Construct, tags: {[ key: string ]: string}): void | **Name** | **Description** | | --- | --- | | getEnvConfig | *No description.* | +| getTags | *No description.* | --- @@ -9154,6 +9402,20 @@ Configurator.getEnvConfig(props: GetEnvConfigProps) --- +##### `getTags` + +```typescript +import { Configurator } from 'aws-ddk-core' + +Configurator.getTags(props: GetTagsProps) +``` + +###### `props`Required + +- *Type:* GetTagsProps + +--- + #### Properties | **Name** | **Type** | **Description** | diff --git a/src/base/stack.ts b/src/base/stack.ts index 5d9461b4..5429993c 100644 --- a/src/base/stack.ts +++ b/src/base/stack.ts @@ -9,7 +9,82 @@ export interface BaseStackProps extends cdk.StackProps { readonly config?: string | object; } +export interface PermissionsBoundaryProps { + readonly environmentId?: string; + readonly prefix?: string; + readonly qualifier?: string; +} + export class BaseStack extends cdk.Stack { + public static createDefaultPermissionsBoundary(scope: Construct, id: string, props: PermissionsBoundaryProps) { + const prefix = props.prefix ?? "ddk"; + const environmentId = props.environmentId ?? "dev"; + const qualifier = props.environmentId ?? "hnb659fds"; + + const policyStatements = [ + new iam.PolicyStatement({ + effect: iam.Effect.DENY, + actions: ["s3:PutAccountPublicAccessBlock"], + resources: ["*"], + }), + new iam.PolicyStatement({ + effect: iam.Effect.DENY, + actions: [ + "iam:CreatePolicyVersion", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:SetDefaultPolicyVersion", + ], + resources: [ + `arn:${cdk.Stack.of(scope).partition}:iam::${ + cdk.Stack.of(scope).account + }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ + cdk.Stack.of(scope).region + }`, + ], + }), + new iam.PolicyStatement({ + effect: iam.Effect.DENY, + actions: ["iam:DeleteRolePermissionsBoundary"], + resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`], + conditions: { + "ForAnyValue:StringEquals": { + "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${ + cdk.Stack.of(scope).account + }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ + cdk.Stack.of(scope).region + }`, + }, + }, + }), + new iam.PolicyStatement({ + effect: iam.Effect.DENY, + actions: ["iam:PutRolePermissionsBoundary"], + resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`], + conditions: { + "ForAnyValue:StringNotEquals": { + "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${ + cdk.Stack.of(scope).account + }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ + cdk.Stack.of(scope).region + }`, + }, + }, + }), + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ["*"], + resources: ["*"], + }), + ]; + return new iam.ManagedPolicy(scope, id, { + statements: policyStatements, + managedPolicyName: `${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ + cdk.Stack.of(scope).region + }`, + description: "AWS-DDK: Deny dangerous actions that could escalate privilege or cause security incident", + }); + } readonly terminationProtection?: boolean | undefined; constructor(scope: Construct, id: string, props: BaseStackProps) { @@ -27,79 +102,3 @@ export class BaseStack extends cdk.Stack { } } } - -export interface PermissionsBoundaryProps { - readonly environmentId?: string; - readonly prefix?: string; - readonly qualifier?: string; -} - -export function createDefaultPermissionsBoundary(scope: Construct, id: string, props: PermissionsBoundaryProps) { - const prefix = props.prefix ?? "ddk"; - const environmentId = props.environmentId ?? "dev"; - const qualifier = props.environmentId ?? "hnb659fds"; - - const policyStatements = [ - new iam.PolicyStatement({ - effect: iam.Effect.DENY, - actions: ["s3:PutAccountPublicAccessBlock"], - resources: ["*"], - }), - new iam.PolicyStatement({ - effect: iam.Effect.DENY, - actions: [ - "iam:CreatePolicyVersion", - "iam:DeletePolicy", - "iam:DeletePolicyVersion", - "iam:SetDefaultPolicyVersion", - ], - resources: [ - `arn:${cdk.Stack.of(scope).partition}:iam::${ - cdk.Stack.of(scope).account - }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ - cdk.Stack.of(scope).region - }`, - ], - }), - new iam.PolicyStatement({ - effect: iam.Effect.DENY, - actions: ["iam:DeleteRolePermissionsBoundary"], - resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`], - conditions: { - "ForAnyValue:StringEquals": { - "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${ - cdk.Stack.of(scope).account - }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ - cdk.Stack.of(scope).region - }`, - }, - }, - }), - new iam.PolicyStatement({ - effect: iam.Effect.DENY, - actions: ["iam:PutRolePermissionsBoundary"], - resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`], - conditions: { - "ForAnyValue:StringNotEquals": { - "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${ - cdk.Stack.of(scope).account - }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ - cdk.Stack.of(scope).region - }`, - }, - }, - }), - new iam.PolicyStatement({ - effect: iam.Effect.ALLOW, - actions: ["*"], - resources: ["*"], - }), - ]; - return new iam.ManagedPolicy(scope, id, { - statements: policyStatements, - managedPolicyName: `${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${ - cdk.Stack.of(scope).region - }`, - description: "AWS-DDK: Deny dangerous actions that could escalate privilege or cause security incident", - }); -} diff --git a/src/config/configurator.ts b/src/config/configurator.ts index a3f0faff..29509602 100644 --- a/src/config/configurator.ts +++ b/src/config/configurator.ts @@ -129,11 +129,26 @@ export interface GetEnvConfigProps { readonly environmentId: string; } +export interface GetTagsProps { + readonly configPath: string; + readonly environmentId?: string; +} + export class Configurator { public static getEnvConfig(props: GetEnvConfigProps): any { const config = getConfig({ config: props.configPath }); return config.environments ? config.environments[props.environmentId] : undefined; } + public static getTags(props: GetTagsProps): any { + const config = getConfig({ config: props.configPath }); + return props.environmentId + ? config.environments + ? config.environments[props.environmentId].tags + : {} + : config.tags + ? config.tags + : {}; + } public readonly config: any; public readonly environmentId?: string; constructor(scope: constructs.Construct, config: string | object, environmentId?: string) { diff --git a/src/pipelines/stage.ts b/src/pipelines/stage.ts index 118aa959..79d9e2ec 100644 --- a/src/pipelines/stage.ts +++ b/src/pipelines/stage.ts @@ -53,7 +53,7 @@ export abstract class DataStage extends Stage { new cloudwatch.Alarm(this, id, { metric: props.metric, comparisonOperator: props.comparisonOperator ?? cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - threshold: props.threshold ?? 5, + threshold: props.threshold ?? 1, evaluationPeriods: props.evaluationPeriods ?? 1, }), ); diff --git a/src/stages/databrew-transform.ts b/src/stages/databrew-transform.ts index ae1194e1..efb4788e 100644 --- a/src/stages/databrew-transform.ts +++ b/src/stages/databrew-transform.ts @@ -1,3 +1,4 @@ +import * as cdk from "aws-cdk-lib"; import * as databrew from "aws-cdk-lib/aws-databrew"; import * as events from "aws-cdk-lib/aws-events"; import * as iam from "aws-cdk-lib/aws-iam"; @@ -12,9 +13,23 @@ export interface DataBrewTransformStageProps extends StateMachineStageProps { readonly jobRoleArn?: string; readonly jobType?: string; readonly datasetName?: string; - readonly recipe?: databrew.CfnJob.RecipeProperty; - readonly outputs?: databrew.CfnJob.OutputProperty[]; readonly createJob?: boolean; + readonly dataCatalogOutputs?: databrew.CfnJob.DataCatalogOutputProperty[]; + readonly databaseOutputs?: databrew.CfnJob.DatabaseOutputProperty[]; + readonly encryptionKeyArn?: string; + readonly encryptionMode?: string; + readonly jobSample?: databrew.CfnJob.JobSampleProperty; + readonly logSubscription?: string; + readonly maxCapacity?: number; + readonly maxRetries?: number; + readonly outputLocation?: databrew.CfnJob.OutputLocationProperty; + readonly outputs?: databrew.CfnJob.OutputProperty[]; + readonly profileConfiguration?: databrew.CfnJob.ProfileConfigurationProperty; + readonly projectName?: string; + readonly recipe?: databrew.CfnJob.RecipeProperty; + readonly tags?: cdk.CfnTag[]; + readonly timeout?: number; + readonly validationConfigurations?: databrew.CfnJob.ValidationConfigurationProperty[]; } export class DataBrewTransformStage extends StateMachineStage { @@ -28,21 +43,34 @@ export class DataBrewTransformStage extends StateMachineStage { constructor(scope: Construct, id: string, props: DataBrewTransformStageProps) { super(scope, id, props); - const { jobName, jobRoleArn, jobType, datasetName, recipe, outputs, createJob } = props; - this.jobName = jobName ? jobName : `${id}-job`; - this.createJob = jobName && !createJob ? false : true; + this.jobName = props.jobName ? props.jobName : `${id}-job`; + this.createJob = props.jobName && !props.createJob ? false : true; if (this.createJob) { - if (!jobType) { + if (!props.jobType) { throw new Error("if 'jobType' is a required property when creating a new DataBrew job"); } this.job = new databrew.CfnJob(this, "DataBrew Job", { name: this.jobName, - roleArn: jobRoleArn ? jobRoleArn : this.createDefaultDataBrewJobRole().roleArn, - type: jobType, - datasetName: datasetName, - recipe: recipe, - outputs: outputs, + roleArn: props.jobRoleArn ? props.jobRoleArn : this.createDefaultDataBrewJobRole().roleArn, + type: props.jobType, + datasetName: props.datasetName, + dataCatalogOutputs: props.dataCatalogOutputs, + databaseOutputs: props.databaseOutputs, + encryptionKeyArn: props.encryptionKeyArn, + encryptionMode: props.encryptionMode, + jobSample: props.jobSample, + logSubscription: props.logSubscription, + maxCapacity: props.maxCapacity, + maxRetries: props.maxRetries, + outputLocation: props.outputLocation, + outputs: props.outputs, + profileConfiguration: props.profileConfiguration, + projectName: props.projectName, + recipe: props.recipe, + tags: props.tags, + timeout: props.timeout, + validationConfigurations: props.validationConfigurations, }); } const startJobRun = new tasks.GlueDataBrewStartJobRun(this, "Start DataBrew Job", { diff --git a/src/stages/sqs-lambda.ts b/src/stages/sqs-lambda.ts index 7f3dfbdf..5f7b8c9a 100644 --- a/src/stages/sqs-lambda.ts +++ b/src/stages/sqs-lambda.ts @@ -48,18 +48,14 @@ export class SqsToLambdaStage extends DataStage { const functionProps: SqsToLambdaStageFunctionProps = props.lambdaFunctionProps; this.function = new lambda.Function(this, "Process Function", { - code: functionProps.code, - runtime: functionProps.runtime, - handler: functionProps.handler, timeout: functionProps.timeout ?? cdk.Duration.seconds(120), memorySize: functionProps.memorySize ?? 256, - layers: functionProps.layers, - role: functionProps.role, environment: { EVENT_SOURCE: eventSource, EVENT_DETAIL_TYPE: eventDetailType, ...(functionProps.environment ?? {}), }, + ...functionProps, }); } else { throw TypeError("'lambdaFunction' or 'lambdaFunctionProps' must be set to instantiate this stage"); diff --git a/test/appflow-ingestion-stage.test.ts b/test/appflow-ingestion-stage.test.ts index 5daaf5fd..610e05bd 100644 --- a/test/appflow-ingestion-stage.test.ts +++ b/test/appflow-ingestion-stage.test.ts @@ -34,7 +34,7 @@ test("AppFlow Ingestion stage creates State Machine, Lambda & Alarm", () => { Namespace: "AWS/States", Period: 300, Statistic: "Sum", - Threshold: 5, + Threshold: 1, }); }); diff --git a/test/base-stack.test.ts b/test/base-stack.test.ts index 334d3b37..94b1bd24 100644 --- a/test/base-stack.test.ts +++ b/test/base-stack.test.ts @@ -4,7 +4,7 @@ import * as cdk from "aws-cdk-lib"; import { Template } from "aws-cdk-lib/assertions"; import * as lambda from "aws-cdk-lib/aws-lambda"; -import { BaseStack, createDefaultPermissionsBoundary, SqsToLambdaStage } from "../src"; +import { BaseStack, SqsToLambdaStage } from "../src"; test("Base Stack No Bootstrap Config", () => { const sampleConfig = { @@ -310,7 +310,11 @@ test("Base Stack Permissions Boundary", () => { test("Test Permissions Boundary Creation and Usage in BaseStack", () => { const app = new cdk.App(); const bootstrapStack = new cdk.Stack(app, "my-bootstrap-stack"); - const permissionsBoundary = createDefaultPermissionsBoundary(bootstrapStack, "DDK Default Permissions Boundary", {}); + const permissionsBoundary = BaseStack.createDefaultPermissionsBoundary( + bootstrapStack, + "DDK Default Permissions Boundary", + {}, + ); const stack = new BaseStack(app, "my-stack", { environmentId: "dev", permissionsBoundaryArn: permissionsBoundary.managedPolicyArn, @@ -337,7 +341,7 @@ test("Test Permissions Boundary Creation and Usage in BaseStack", () => { test("Permissions Boundary Creation Full", () => { const app = new cdk.App(); const bootstrapStack = new cdk.Stack(app, "my-bootstrap-stack"); - createDefaultPermissionsBoundary(bootstrapStack, "DDK Default Permissions Boundary", { + BaseStack.createDefaultPermissionsBoundary(bootstrapStack, "DDK Default Permissions Boundary", { prefix: "custom", environmentId: "stage", qualifier: "abcdefgh", diff --git a/test/config.test.ts b/test/config.test.ts index 866c169a..e1b32778 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -467,3 +467,12 @@ test("Get Env Config Static Method", () => { assert(config === expectedDevConfig); assert(nullConfig === undefined); }); + +test("Get Tags", () => { + const devTags = Configurator.getTags({ configPath: "./test/test-config.yaml", environmentId: "dev" }); + const prodTags = Configurator.getTags({ configPath: "./test/test-config.yaml", environmentId: "prod" }); + const globalTags = Configurator.getTags({ configPath: "./test/test-config.yaml" }); + assert(devTags.CostCenter === "2014"); + assert(prodTags.CostCenter === "2015"); + assert(globalTags["global:foo"] === "bar"); +}); diff --git a/test/databrew-transform-stage.test.ts b/test/databrew-transform-stage.test.ts index cb892af6..5eae078b 100644 --- a/test/databrew-transform-stage.test.ts +++ b/test/databrew-transform-stage.test.ts @@ -24,7 +24,7 @@ test("DataBrew Transfrom stage creates State Machine & Alarm", () => { Namespace: "AWS/States", Period: 300, Statistic: "Sum", - Threshold: 5, + Threshold: 1, }); }); @@ -108,3 +108,38 @@ test("DataBrew Transform must have 'jobType' if 'createJob' is enabled or no exi }); }).toThrowError("if 'jobType' is a required property when creating a new DataBrew job"); }); + +test("DataBrew Transfrom stage additional properties", () => { + const stack = new cdk.Stack(); + + new DataBrewTransformStage(stack, "databrew-transform", { + jobName: "dummy-job", + jobType: "PROFILE", + createJob: true, + maxCapacity: 2, + maxRetries: 2, + encryptionMode: "SSE-S3", + }); + + const template = Template.fromStack(stack); + template.hasResourceProperties("AWS::DataBrew::Job", { + Name: "dummy-job", + Type: "PROFILE", + MaxCapacity: 2, + MaxRetries: 2, + }); + template.hasResourceProperties("AWS::StepFunctions::StateMachine", { + DefinitionString: { + "Fn::Join": ["", Match.arrayWith([Match.stringLikeRegexp("Start DataBrew Job")])], + }, + }); + template.hasResourceProperties("AWS::CloudWatch::Alarm", { + ComparisonOperator: "GreaterThanThreshold", + EvaluationPeriods: 1, + MetricName: "ExecutionsFailed", + Namespace: "AWS/States", + Period: 300, + Statistic: "Sum", + Threshold: 1, + }); +}); diff --git a/test/kinesis-s3-stage.test.ts b/test/kinesis-s3-stage.test.ts index 8d6a3394..422e96bc 100644 --- a/test/kinesis-s3-stage.test.ts +++ b/test/kinesis-s3-stage.test.ts @@ -51,7 +51,7 @@ test("FirehoseToS3Stage creates Firehose DeliveryStream and S3 Bucket", () => { template.hasResourceProperties("AWS::CloudWatch::Alarm", { MetricName: "DeliveryToS3.DataFreshness", - Threshold: 5, + Threshold: 1, EvaluationPeriods: 1, Dimensions: Match.arrayWith([ Match.objectLike({ diff --git a/test/sqs-lambda-stage.test.ts b/test/sqs-lambda-stage.test.ts index aa3c38d1..404c984b 100644 --- a/test/sqs-lambda-stage.test.ts +++ b/test/sqs-lambda-stage.test.ts @@ -205,6 +205,7 @@ test("SQSToLambdaStage additional properties", () => { new SqsToLambdaStage(stack, "Stage", { lambdaFunctionProps: { + functionName: "dummy-function", code: lambda.Code.fromAsset(path.join(__dirname, "/../src/")), handler: "commons.handlers.lambda_handler", runtime: lambda.Runtime.PYTHON_3_8, @@ -222,6 +223,7 @@ test("SQSToLambdaStage additional properties", () => { template.hasResourceProperties("AWS::Lambda::Function", { Runtime: "python3.8", + FunctionName: "dummy-function", }); template.resourceCountIs("AWS::SQS::Queue", 2); template.hasResourceProperties("AWS::SQS::Queue", { From 50f05d2f7d431f34a2b277c4d391b33218646e1a Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Tue, 4 Apr 2023 15:33:08 -0700 Subject: [PATCH 2/8] sqs props --- src/stages/sqs-lambda.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stages/sqs-lambda.ts b/src/stages/sqs-lambda.ts index 5f7b8c9a..245026ea 100644 --- a/src/stages/sqs-lambda.ts +++ b/src/stages/sqs-lambda.ts @@ -88,6 +88,7 @@ export class SqsToLambdaStage extends DataStage { } : undefined, fifo: props.sqsQueueProps?.fifo ? props.sqsQueueProps?.fifo : undefined, + ...props.sqsQueueProps }); } From 1adeb18ade8b76888f31a0e537245b442b846097 Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Tue, 4 Apr 2023 15:49:20 -0700 Subject: [PATCH 3/8] sqs props test --- src/stages/sqs-lambda.ts | 2 +- test/sqs-lambda-stage.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stages/sqs-lambda.ts b/src/stages/sqs-lambda.ts index 245026ea..2d750109 100644 --- a/src/stages/sqs-lambda.ts +++ b/src/stages/sqs-lambda.ts @@ -88,7 +88,7 @@ export class SqsToLambdaStage extends DataStage { } : undefined, fifo: props.sqsQueueProps?.fifo ? props.sqsQueueProps?.fifo : undefined, - ...props.sqsQueueProps + ...props.sqsQueueProps, }); } diff --git a/test/sqs-lambda-stage.test.ts b/test/sqs-lambda-stage.test.ts index 404c984b..3993f6e9 100644 --- a/test/sqs-lambda-stage.test.ts +++ b/test/sqs-lambda-stage.test.ts @@ -214,6 +214,7 @@ test("SQSToLambdaStage additional properties", () => { maxReceiveCount: 2, dlqEnabled: true, sqsQueueProps: { + queueName: "dummy-queue.fifo", fifo: true, visibilityTimeout: cdk.Duration.minutes(5), }, @@ -228,5 +229,6 @@ test("SQSToLambdaStage additional properties", () => { template.resourceCountIs("AWS::SQS::Queue", 2); template.hasResourceProperties("AWS::SQS::Queue", { VisibilityTimeout: 300, + QueueName: "dummy-queue.fifo", }); }); From b086358e13bab43c93443ebd7081ee8852f26ccb Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Wed, 5 Apr 2023 10:45:14 -0700 Subject: [PATCH 4/8] adding ruleName support --- src/pipelines/pipelines.ts | 3 ++- test/data-pipelines.test.ts | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/pipelines/pipelines.ts b/src/pipelines/pipelines.ts index 1eb4e21f..4825511f 100644 --- a/src/pipelines/pipelines.ts +++ b/src/pipelines/pipelines.ts @@ -48,7 +48,7 @@ export class DataPipeline extends Construct { const skipRule = props.skipRule ?? false; if (props.overrideRule) { - this.addRule({ overrideRule: props.overrideRule, schedule: props.schedule }); + this.addRule({ overrideRule: props.overrideRule, schedule: props.schedule, ruleName: props.ruleName }); } else if (this.previousStage && skipRule === false) { if (stage.targets === undefined) { throw new Error( @@ -61,6 +61,7 @@ export class DataPipeline extends Construct { eventPattern: this.previousStage?.eventPattern, eventTargets: stage.targets, schedule: props.schedule, + ruleName: props.ruleName, }); } diff --git a/test/data-pipelines.test.ts b/test/data-pipelines.test.ts index 360fdd06..4359540d 100644 --- a/test/data-pipelines.test.ts +++ b/test/data-pipelines.test.ts @@ -181,3 +181,46 @@ test("DataPipeline with skip rule", () => { const template = Template.fromStack(stack); template.resourceCountIs("AWS::Events::Rule", 0); }); + +test("DataPipeline Rule Name", () => { + const stack = new cdk.Stack(); + + const bucket = new s3.Bucket(stack, "Bucket"); + + const firehoseToS3Stage = new FirehoseToS3Stage(stack, "Firehose To S3 Stage", { s3Bucket: bucket }); + + const sqsToLambdaStage = new SqsToLambdaStage(stack, "SQS To Lambda Stage 2", { + lambdaFunctionProps: { + code: lambda.Code.fromAsset(path.join(__dirname, "/../src/")), + handler: "commons.handlers.lambda_handler", + memorySize: 512, + runtime: lambda.Runtime.PYTHON_3_9, + layers: [ + lambda.LayerVersion.fromLayerVersionArn(stack, "Layer", "arn:aws:lambda:us-east-1:222222222222:layer:dummy:1"), + ], + }, + }); + + const pipeline = new DataPipeline(stack, "Pipeline", {}); + + pipeline.addStage({ stage: firehoseToS3Stage }).addStage({ stage: sqsToLambdaStage, ruleName: "foobar" }); + + const template = Template.fromStack(stack); + + template.resourceCountIs("AWS::Events::Rule", 1); + template.hasResourceProperties("AWS::Events::Rule", { + State: "ENABLED", + Name: "foobar", + EventPattern: Match.objectLike({ + "detail-type": firehoseToS3Stage.eventPattern?.detailType, + source: firehoseToS3Stage.eventPattern?.source, + }), + Targets: Match.arrayEquals([ + Match.objectLike({ + Arn: { + "Fn::GetAtt": [stack.resolve((sqsToLambdaStage.queue.node.defaultChild as cdk.CfnElement).logicalId), "Arn"], + }, + }), + ]), + }); +}); From dbc73b3c6cafa85597a0fdbab643e75e57bcf399 Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Wed, 5 Apr 2023 11:29:42 -0700 Subject: [PATCH 5/8] firehose stage escape hatches --- API.md | 109 +++++++++++++++++++++++++++++++++- src/stages/kinesis-s3.ts | 33 +++++++--- test/kinesis-s3-stage.test.ts | 45 ++++++++++++++ 3 files changed, 176 insertions(+), 11 deletions(-) diff --git a/API.md b/API.md index f7c068ac..559847eb 100644 --- a/API.md +++ b/API.md @@ -6886,6 +6886,89 @@ public readonly alarmsEnabled: boolean; --- +### DeliveryStreamProps + +#### Initializer + +```typescript +import { DeliveryStreamProps } from 'aws-ddk-core' + +const deliveryStreamProps: DeliveryStreamProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| deliveryStreamName | string | *No description.* | +| destinations | @aws-cdk/aws-kinesisfirehose-alpha.IDestination[] | *No description.* | +| encryption | @aws-cdk/aws-kinesisfirehose-alpha.StreamEncryption | *No description.* | +| encryptionKey | aws-cdk-lib.aws_kms.IKey | *No description.* | +| role | aws-cdk-lib.aws_iam.IRole | *No description.* | +| sourceStream | aws-cdk-lib.aws_kinesis.IStream | *No description.* | + +--- + +##### `deliveryStreamName`Optional + +```typescript +public readonly deliveryStreamName: string; +``` + +- *Type:* string + +--- + +##### `destinations`Optional + +```typescript +public readonly destinations: IDestination[]; +``` + +- *Type:* @aws-cdk/aws-kinesisfirehose-alpha.IDestination[] + +--- + +##### `encryption`Optional + +```typescript +public readonly encryption: StreamEncryption; +``` + +- *Type:* @aws-cdk/aws-kinesisfirehose-alpha.StreamEncryption + +--- + +##### `encryptionKey`Optional + +```typescript +public readonly encryptionKey: IKey; +``` + +- *Type:* aws-cdk-lib.aws_kms.IKey + +--- + +##### `role`Optional + +```typescript +public readonly role: IRole; +``` + +- *Type:* aws-cdk-lib.aws_iam.IRole + +--- + +##### `sourceStream`Optional + +```typescript +public readonly sourceStream: IStream; +``` + +- *Type:* aws-cdk-lib.aws_kinesis.IStream + +--- + ### EventStageProps #### Initializer @@ -6943,10 +7026,12 @@ const firehoseToS3StageProps: FirehoseToS3StageProps = { ... } | name | string | *No description.* | | alarmsEnabled | boolean | *No description.* | | dataOutputPrefix | string | *No description.* | +| dataStream | aws-cdk-lib.aws_kinesis.Stream | *No description.* | | dataStreamEnabled | boolean | *No description.* | | deliveryStreamDataFreshnessErrorsAlarmThreshold | number | *No description.* | | deliveryStreamDataFreshnessErrorsEvaluationPeriods | number | *No description.* | -| firehoseDeliveryStreamProps | @aws-cdk/aws-kinesisfirehose-alpha.DeliveryStreamProps | *No description.* | +| firehoseDeliveryStream | @aws-cdk/aws-kinesisfirehose-alpha.DeliveryStream | *No description.* | +| firehoseDeliveryStreamProps | DeliveryStreamProps | *No description.* | | kinesisFirehoseDestinationsS3BucketProps | @aws-cdk/aws-kinesisfirehose-destinations-alpha.S3BucketProps | *No description.* | | s3Bucket | aws-cdk-lib.aws_s3.IBucket | *No description.* | | s3BucketProps | aws-cdk-lib.aws_s3.BucketProps | *No description.* | @@ -6993,6 +7078,16 @@ public readonly dataOutputPrefix: string; --- +##### `dataStream`Optional + +```typescript +public readonly dataStream: Stream; +``` + +- *Type:* aws-cdk-lib.aws_kinesis.Stream + +--- + ##### `dataStreamEnabled`Optional ```typescript @@ -7023,13 +7118,23 @@ public readonly deliveryStreamDataFreshnessErrorsEvaluationPeriods: number; --- +##### `firehoseDeliveryStream`Optional + +```typescript +public readonly firehoseDeliveryStream: DeliveryStream; +``` + +- *Type:* @aws-cdk/aws-kinesisfirehose-alpha.DeliveryStream + +--- + ##### `firehoseDeliveryStreamProps`Optional ```typescript public readonly firehoseDeliveryStreamProps: DeliveryStreamProps; ``` -- *Type:* @aws-cdk/aws-kinesisfirehose-alpha.DeliveryStreamProps +- *Type:* DeliveryStreamProps --- diff --git a/src/stages/kinesis-s3.ts b/src/stages/kinesis-s3.ts index a4d7b7d3..7bf27dba 100644 --- a/src/stages/kinesis-s3.ts +++ b/src/stages/kinesis-s3.ts @@ -2,21 +2,33 @@ import * as firehose from "@aws-cdk/aws-kinesisfirehose-alpha"; import * as destinations from "@aws-cdk/aws-kinesisfirehose-destinations-alpha"; import * as cdk from "aws-cdk-lib"; import * as events from "aws-cdk-lib/aws-events"; +import * as iam from "aws-cdk-lib/aws-iam"; import * as kinesis from "aws-cdk-lib/aws-kinesis"; +import * as kms from "aws-cdk-lib/aws-kms"; import * as s3 from "aws-cdk-lib/aws-s3"; import { Construct } from "constructs"; import { overrideProps } from "../core"; import { S3Factory } from "../core/s3-factory"; import { DataStage, DataStageProps } from "../pipelines/stage"; +export interface DeliveryStreamProps { + readonly destinations?: firehose.IDestination[]; + readonly deliveryStreamName?: string; + readonly encryption?: firehose.StreamEncryption; + readonly encryptionKey?: kms.IKey; + readonly role?: iam.IRole; + readonly sourceStream?: kinesis.IStream; +} + export interface FirehoseToS3StageProps extends DataStageProps { readonly s3Bucket?: s3.IBucket; readonly s3BucketProps?: s3.BucketProps; - readonly firehoseDeliveryStreamProps?: firehose.DeliveryStreamProps; + readonly firehoseDeliveryStream?: firehose.DeliveryStream; + readonly firehoseDeliveryStreamProps?: DeliveryStreamProps; readonly kinesisFirehoseDestinationsS3BucketProps?: destinations.S3BucketProps; - readonly dataOutputPrefix?: string; readonly dataStreamEnabled?: boolean; + readonly dataStream?: kinesis.Stream; readonly deliveryStreamDataFreshnessErrorsAlarmThreshold?: number; readonly deliveryStreamDataFreshnessErrorsEvaluationPeriods?: number; } @@ -43,9 +55,10 @@ export class FirehoseToS3Stage extends DataStage { throw TypeError("'s3Bucket' or 's3BucketProps' must be set to instantiate this stage"); } - const dataStreamEnabled = props.dataStreamEnabled ?? false; - if (dataStreamEnabled == true) { + if (props.dataStreamEnabled == true && !props.dataStream) { this.dataStream = new kinesis.Stream(this, "Data Stream", {}); + } else if (props.dataStreamEnabled != false && props.dataStream) { + this.dataStream = props.dataStream; } const destinationsBucketProps = overrideProps( @@ -59,11 +72,13 @@ export class FirehoseToS3Stage extends DataStage { dataOutputPrefix: props.dataOutputPrefix, }, ); - this.deliveryStream = new firehose.DeliveryStream(this, "Delivery Stream", { - destinations: [new destinations.S3Bucket(this.bucket, destinationsBucketProps)], - sourceStream: this.dataStream, - ...props.firehoseDeliveryStreamProps, - }); + this.deliveryStream = props.firehoseDeliveryStream + ? props.firehoseDeliveryStream + : new firehose.DeliveryStream(this, "Delivery Stream", { + destinations: [new destinations.S3Bucket(this.bucket, destinationsBucketProps)], + sourceStream: this.dataStream, + ...props.firehoseDeliveryStreamProps, + }); const dataOutputPrefix: string = destinationsBucketProps.dataOutputPrefix; this.addAlarm("Data Freshness Errors", { diff --git a/test/kinesis-s3-stage.test.ts b/test/kinesis-s3-stage.test.ts index 422e96bc..c10b2363 100644 --- a/test/kinesis-s3-stage.test.ts +++ b/test/kinesis-s3-stage.test.ts @@ -1,6 +1,9 @@ +import * as firehose from "@aws-cdk/aws-kinesisfirehose-alpha"; +import * as destinations from "@aws-cdk/aws-kinesisfirehose-destinations-alpha"; import * as cdk from "aws-cdk-lib"; import { Size } from "aws-cdk-lib"; import { Match, Template } from "aws-cdk-lib/assertions"; +import * as kinesis from "aws-cdk-lib/aws-kinesis"; import { Bucket } from "aws-cdk-lib/aws-s3"; import { FirehoseToS3Stage } from "../src"; @@ -95,6 +98,48 @@ test("FirehoseToS3Stage uses S3 Bucket and creates Kinesis DataStream", () => { }); }); +test("FirehoseToS3Stage uses existing S3 Bucket and Kinesis DataStream", () => { + const stack = new cdk.Stack(); + + const stage = new FirehoseToS3Stage(stack, "Stage", { + s3Bucket: new Bucket(stack, "Bucket"), + dataStream: new kinesis.Stream(stack, "Stream"), + deliveryStreamDataFreshnessErrorsAlarmThreshold: 10, + }); + + const template = Template.fromStack(stack); + + template.resourceCountIs("AWS::S3::Bucket", 1); + template.resourceCountIs("AWS::Kinesis::Stream", 1); + + template.hasResourceProperties("AWS::KinesisFirehose::DeliveryStream", { + KinesisStreamSourceConfiguration: Match.objectLike({ + KinesisStreamARN: Match.objectLike({ + "Fn::GetAtt": Match.arrayWith([ + stack.resolve((stage.dataStream?.node.defaultChild as cdk.CfnElement).logicalId), + "Arn", + ]), + }), + }), + }); +}); + +test("FirehoseToS3Stage uses existing S3 Bucket and Firehose DeliveryStream", () => { + const stack = new cdk.Stack(); + const s3Bucket = new Bucket(stack, "Bucket"); + new FirehoseToS3Stage(stack, "Stage", { + s3Bucket: s3Bucket, + firehoseDeliveryStream: new firehose.DeliveryStream(stack, "DeliveryStream", { + destinations: [new destinations.S3Bucket(s3Bucket, {})], + }), + }); + + const template = Template.fromStack(stack); + + template.resourceCountIs("AWS::S3::Bucket", 1); + template.resourceCountIs("AWS::KinesisFirehose::DeliveryStream", 1); +}); + test("FirehoseToS3Stage uses dataOutputPrefix in Firehose DeliveryStream", () => { const stack = new cdk.Stack(); const bucket = new Bucket(stack, "Bucket"); From d76a89292a10bb90b0c671d66ffcd0cf4c59e281 Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Wed, 5 Apr 2023 12:37:44 -0700 Subject: [PATCH 6/8] glue crawler props refactor --- API.md | 33 ++++++++++++++++++++++++++ src/stages/glue-transform.ts | 19 ++++++++++++--- test/glue-transform-stage.test.ts | 39 +++++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/API.md b/API.md index 559847eb..3db42e36 100644 --- a/API.md +++ b/API.md @@ -7398,12 +7398,15 @@ const glueTransformStageProps: GlueTransformStageProps = { ... } | crawlerAllowFailure | boolean | *No description.* | | crawlerName | string | *No description.* | | crawlerProps | aws-cdk-lib.aws_glue.CfnCrawlerProps | *No description.* | +| crawlerRole | string | *No description.* | +| databaseName | string | *No description.* | | jobName | string | *No description.* | | jobProps | @aws-cdk/aws-glue-alpha.JobProps | *No description.* | | jobRunArgs | {[ key: string ]: any} | *No description.* | | stateMachineRetryBackoffRate | number | *No description.* | | stateMachineRetryInterval | aws-cdk-lib.Duration | *No description.* | | stateMachineRetryMaxAttempts | number | *No description.* | +| targets | aws-cdk-lib.aws_glue.CfnCrawler.TargetsProperty | *No description.* | --- @@ -7517,6 +7520,26 @@ public readonly crawlerProps: CfnCrawlerProps; --- +##### `crawlerRole`Optional + +```typescript +public readonly crawlerRole: string; +``` + +- *Type:* string + +--- + +##### `databaseName`Optional + +```typescript +public readonly databaseName: string; +``` + +- *Type:* string + +--- + ##### `jobName`Optional ```typescript @@ -7577,6 +7600,16 @@ public readonly stateMachineRetryMaxAttempts: number; --- +##### `targets`Optional + +```typescript +public readonly targets: TargetsProperty; +``` + +- *Type:* aws-cdk-lib.aws_glue.CfnCrawler.TargetsProperty + +--- + ### PermissionsBoundaryProps #### Initializer diff --git a/src/stages/glue-transform.ts b/src/stages/glue-transform.ts index 8ae4d34e..55142295 100644 --- a/src/stages/glue-transform.ts +++ b/src/stages/glue-transform.ts @@ -13,6 +13,9 @@ export interface GlueTransformStageProps extends StateMachineStageProps { readonly jobProps?: glue_alpha.JobProps; readonly jobRunArgs?: { [key: string]: any }; readonly crawlerName?: string; + readonly crawlerRole?: string; + readonly databaseName?: string; + readonly targets?: glue.CfnCrawler.TargetsProperty; readonly crawlerProps?: glue.CfnCrawlerProps; readonly crawlerAllowFailure?: boolean; readonly stateMachineRetryMaxAttempts?: number; @@ -93,11 +96,21 @@ export class GlueTransformStage extends StateMachineStage { } private getCrawler(props: GlueTransformStageProps): glue.CfnCrawler { - if (!props.crawlerProps) { - throw TypeError("'crawlerName' or 'crawlerProps' must be set to instantiate this stage"); + const role = props.crawlerRole ?? props.crawlerProps?.role; + if (!role) { + throw TypeError("Crawler Role must be set either by 'crawlerRole' or 'crawlerProps.role"); } - const crawler = new glue.CfnCrawler(this, "Crawler", props.crawlerProps); + const targets = props.targets ?? props.crawlerProps?.targets; + if (!targets) { + throw TypeError("Crawler Targets must be set either by 'targets' or 'crawlerProps.targets"); + } + + const crawler = new glue.CfnCrawler(this, "Crawler", { + role: role, + databaseName: props.databaseName, + targets: targets, + }); return crawler; } } diff --git a/test/glue-transform-stage.test.ts b/test/glue-transform-stage.test.ts index bcd17687..a92be247 100644 --- a/test/glue-transform-stage.test.ts +++ b/test/glue-transform-stage.test.ts @@ -75,6 +75,31 @@ test("GlueTransformStage stage creates Glue Crawler", () => { }); }); +test("GlueTransformStage stage creates Glue Crawler 2", () => { + const stack = new cdk.Stack(); + + new GlueTransformStage(stack, "glue-transform", { + jobName: "myJob", + crawlerRole: "role", + targets: { + s3Targets: [ + { + path: "s3://my-bucket/crawl-path", + }, + ], + }, + }); + + const template = Template.fromStack(stack); + template.resourceCountIs("AWS::Glue::Job", 0); + template.hasResourceProperties("AWS::Glue::Crawler", { + Role: "role", + Targets: { + S3Targets: [{ Path: "s3://my-bucket/crawl-path" }], + }, + }); +}); + test("GlueTranformStage must have 'jobName' or 'jobProps' set", () => { const stack = new cdk.Stack(); expect(() => { @@ -82,13 +107,23 @@ test("GlueTranformStage must have 'jobName' or 'jobProps' set", () => { }).toThrowError("'jobName' or 'jobProps' must be set to instantiate this stage"); }); -test("GlueTranformStage must have 'crawlerName' or 'crawlerProps' set", () => { +test("GlueTranformStage must set crawler role ", () => { + const stack = new cdk.Stack(); + expect(() => { + new GlueTransformStage(stack, "Stage", { + jobName: "myJob", + }); + }).toThrowError("Crawler Role must be set either by 'crawlerRole' or 'crawlerProps.role"); +}); + +test("GlueTranformStage must set crawler targets", () => { const stack = new cdk.Stack(); expect(() => { new GlueTransformStage(stack, "Stage", { jobName: "myJob", + crawlerRole: "arn:aws:iam::role/dummy-role", }); - }).toThrowError("'crawlerName' or 'crawlerProps' must be set to instantiate this stage"); + }).toThrowError("Crawler Targets must be set either by 'targets' or 'crawlerProps.targets"); }); test("GlueTransformStage retry settings", () => { From 0dd8e919c3dca275ec73424f8ffbed8be1bad973 Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Wed, 5 Apr 2023 13:02:22 -0700 Subject: [PATCH 7/8] glue factory duplication --- src/core/glue-factory.ts | 13 ++++++++----- src/stages/glue-transform.ts | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core/glue-factory.ts b/src/core/glue-factory.ts index b3d5e1fe..0b74ac89 100644 --- a/src/core/glue-factory.ts +++ b/src/core/glue-factory.ts @@ -5,15 +5,18 @@ import { overrideProps } from "./utils"; export class GlueFactory { public static job(scope: Construct, id: string, props: glue.JobProps) { + const securityConfiguration = !props.securityConfiguration + ? new glue.SecurityConfiguration(scope, `${id}-security-configuration`, { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, + }) + : undefined; const defaultProps: Partial = { maxConcurrentRuns: 1, maxRetries: 1, timeout: cdk.Duration.hours(10), - securityConfiguration: new glue.SecurityConfiguration(scope, `${id}-security-configuration`, { - s3Encryption: { - mode: glue.S3EncryptionMode.S3_MANAGED, - }, - }), + securityConfiguration: securityConfiguration, }; const mergedProps = overrideProps(defaultProps, props); diff --git a/src/stages/glue-transform.ts b/src/stages/glue-transform.ts index 55142295..3703694d 100644 --- a/src/stages/glue-transform.ts +++ b/src/stages/glue-transform.ts @@ -110,6 +110,7 @@ export class GlueTransformStage extends StateMachineStage { role: role, databaseName: props.databaseName, targets: targets, + ...props.crawlerProps, }); return crawler; } From b2b79a43cab6af63fd115d93e66ad851da743409 Mon Sep 17 00:00:00 2001 From: lucas hanson Date: Wed, 5 Apr 2023 15:21:42 -0700 Subject: [PATCH 8/8] typing --- src/base/stack.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/base/stack.ts b/src/base/stack.ts index 5429993c..2008eeb7 100644 --- a/src/base/stack.ts +++ b/src/base/stack.ts @@ -16,7 +16,11 @@ export interface PermissionsBoundaryProps { } export class BaseStack extends cdk.Stack { - public static createDefaultPermissionsBoundary(scope: Construct, id: string, props: PermissionsBoundaryProps) { + public static createDefaultPermissionsBoundary( + scope: Construct, + id: string, + props: PermissionsBoundaryProps, + ): iam.IManagedPolicy { const prefix = props.prefix ?? "ddk"; const environmentId = props.environmentId ?? "dev"; const qualifier = props.environmentId ?? "hnb659fds";