fix: Beta Testing Fixes

src/base/stack.ts

@@ -9,7 +9,86 @@ export interface BaseStackProps extends cdk.StackProps {
   readonly config?: string | object;
 }
 
+export interface PermissionsBoundaryProps {
+  readonly environmentId?: string;
+  readonly prefix?: string;
+  readonly qualifier?: string;
+}
+
 export class BaseStack extends cdk.Stack {
+  public static createDefaultPermissionsBoundary(
+    scope: Construct,
+    id: string,
+    props: PermissionsBoundaryProps,
+  ): iam.IManagedPolicy {
+    const prefix = props.prefix ?? "ddk";
+    const environmentId = props.environmentId ?? "dev";
+    const qualifier = props.environmentId ?? "hnb659fds";
+
+    const policyStatements = [
+      new iam.PolicyStatement({
+        effect: iam.Effect.DENY,
+        actions: ["s3:PutAccountPublicAccessBlock"],
+        resources: ["*"],
+      }),
+      new iam.PolicyStatement({
+        effect: iam.Effect.DENY,
+        actions: [
+          "iam:CreatePolicyVersion",
+          "iam:DeletePolicy",
+          "iam:DeletePolicyVersion",
+          "iam:SetDefaultPolicyVersion",
+        ],
+        resources: [
+          `arn:${cdk.Stack.of(scope).partition}:iam::${
+            cdk.Stack.of(scope).account
+          }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
+            cdk.Stack.of(scope).region
+          }`,
+        ],
+      }),
+      new iam.PolicyStatement({
+        effect: iam.Effect.DENY,
+        actions: ["iam:DeleteRolePermissionsBoundary"],
+        resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`],
+        conditions: {
+          "ForAnyValue:StringEquals": {
+            "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${
+              cdk.Stack.of(scope).account
+            }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
+              cdk.Stack.of(scope).region
+            }`,
+          },
+        },
+      }),
+      new iam.PolicyStatement({
+        effect: iam.Effect.DENY,
+        actions: ["iam:PutRolePermissionsBoundary"],
+        resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`],
+        conditions: {
+          "ForAnyValue:StringNotEquals": {
+            "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${
+              cdk.Stack.of(scope).account
+            }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
+              cdk.Stack.of(scope).region
+            }`,
+          },
+        },
+      }),
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: ["*"],
+        resources: ["*"],
+      }),
+    ];
+    return new iam.ManagedPolicy(scope, id, {
+      statements: policyStatements,
+      managedPolicyName: `${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
+        cdk.Stack.of(scope).region
+      }`,
+      description: "AWS-DDK: Deny dangerous actions that could escalate privilege or cause security incident",
+    });
+  }
   readonly terminationProtection?: boolean | undefined;
 
   constructor(scope: Construct, id: string, props: BaseStackProps) {
@@ -27,79 +106,3 @@ export class BaseStack extends cdk.Stack {
     }
   }
 }
-
-export interface PermissionsBoundaryProps {
-  readonly environmentId?: string;
-  readonly prefix?: string;
-  readonly qualifier?: string;
-}
-
-export function createDefaultPermissionsBoundary(scope: Construct, id: string, props: PermissionsBoundaryProps) {
-  const prefix = props.prefix ?? "ddk";
-  const environmentId = props.environmentId ?? "dev";
-  const qualifier = props.environmentId ?? "hnb659fds";
-
-  const policyStatements = [
-    new iam.PolicyStatement({
-      effect: iam.Effect.DENY,
-      actions: ["s3:PutAccountPublicAccessBlock"],
-      resources: ["*"],
-    }),
-    new iam.PolicyStatement({
-      effect: iam.Effect.DENY,
-      actions: [
-        "iam:CreatePolicyVersion",
-        "iam:DeletePolicy",
-        "iam:DeletePolicyVersion",
-        "iam:SetDefaultPolicyVersion",
-      ],
-      resources: [
-        `arn:${cdk.Stack.of(scope).partition}:iam::${
-          cdk.Stack.of(scope).account
-        }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
-          cdk.Stack.of(scope).region
-        }`,
-      ],
-    }),
-    new iam.PolicyStatement({
-      effect: iam.Effect.DENY,
-      actions: ["iam:DeleteRolePermissionsBoundary"],
-      resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`],
-      conditions: {
-        "ForAnyValue:StringEquals": {
-          "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${
-            cdk.Stack.of(scope).account
-          }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
-            cdk.Stack.of(scope).region
-          }`,
-        },
-      },
-    }),
-    new iam.PolicyStatement({
-      effect: iam.Effect.DENY,
-      actions: ["iam:PutRolePermissionsBoundary"],
-      resources: [`arn:${cdk.Stack.of(scope).partition}:iam::${cdk.Stack.of(scope).account}:role/*`],
-      conditions: {
-        "ForAnyValue:StringNotEquals": {
-          "iam:PermissionsBoundary": `arn:${cdk.Stack.of(scope).partition}:iam::${
-            cdk.Stack.of(scope).account
-          }:policy/${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
-            cdk.Stack.of(scope).region
-          }`,
-        },
-      },
-    }),
-    new iam.PolicyStatement({
-      effect: iam.Effect.ALLOW,
-      actions: ["*"],
-      resources: ["*"],
-    }),
-  ];
-  return new iam.ManagedPolicy(scope, id, {
-    statements: policyStatements,
-    managedPolicyName: `${prefix}-${environmentId}-${qualifier}-permissions-boundary-${cdk.Stack.of(scope).account}-${
-      cdk.Stack.of(scope).region
-    }`,
-    description: "AWS-DDK: Deny dangerous actions that could escalate privilege or cause security incident",
-  });
-}

src/config/configurator.ts

@@ -129,11 +129,26 @@ export interface GetEnvConfigProps {
   readonly environmentId: string;
 }
 
+export interface GetTagsProps {
+  readonly configPath: string;
+  readonly environmentId?: string;
+}
+
 export class Configurator {
   public static getEnvConfig(props: GetEnvConfigProps): any {
     const config = getConfig({ config: props.configPath });
     return config.environments ? config.environments[props.environmentId] : undefined;
   }
+  public static getTags(props: GetTagsProps): any {
+    const config = getConfig({ config: props.configPath });
+    return props.environmentId
+      ? config.environments
+        ? config.environments[props.environmentId].tags
+        : {}
+      : config.tags
+      ? config.tags
+      : {};
+  }
   public readonly config: any;
   public readonly environmentId?: string;
   constructor(scope: constructs.Construct, config: string | object, environmentId?: string) {

src/core/glue-factory.ts

@@ -5,15 +5,18 @@ import { overrideProps } from "./utils";
 
 export class GlueFactory {
   public static job(scope: Construct, id: string, props: glue.JobProps) {
+    const securityConfiguration = !props.securityConfiguration
+      ? new glue.SecurityConfiguration(scope, `${id}-security-configuration`, {
+          s3Encryption: {
+            mode: glue.S3EncryptionMode.S3_MANAGED,
+          },
+        })
+      : undefined;
     const defaultProps: Partial<glue.JobProps> = {
       maxConcurrentRuns: 1,
       maxRetries: 1,
       timeout: cdk.Duration.hours(10),
-      securityConfiguration: new glue.SecurityConfiguration(scope, `${id}-security-configuration`, {
-        s3Encryption: {
-          mode: glue.S3EncryptionMode.S3_MANAGED,
-        },
-      }),
+      securityConfiguration: securityConfiguration,
     };
 
     const mergedProps = overrideProps(defaultProps, props);

src/pipelines/pipelines.ts

@@ -48,7 +48,7 @@ export class DataPipeline extends Construct {
     const skipRule = props.skipRule ?? false;
 
     if (props.overrideRule) {
-      this.addRule({ overrideRule: props.overrideRule, schedule: props.schedule });
+      this.addRule({ overrideRule: props.overrideRule, schedule: props.schedule, ruleName: props.ruleName });
     } else if (this.previousStage && skipRule === false) {
       if (stage.targets === undefined) {
         throw new Error(
@@ -61,6 +61,7 @@ export class DataPipeline extends Construct {
         eventPattern: this.previousStage?.eventPattern,
         eventTargets: stage.targets,
         schedule: props.schedule,
+        ruleName: props.ruleName,
       });
     }
 

src/pipelines/stage.ts

@@ -53,7 +53,7 @@ export abstract class DataStage extends Stage {
         new cloudwatch.Alarm(this, id, {
           metric: props.metric,
           comparisonOperator: props.comparisonOperator ?? cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
-          threshold: props.threshold ?? 5,
+          threshold: props.threshold ?? 1,
           evaluationPeriods: props.evaluationPeriods ?? 1,
         }),
       );

src/stages/databrew-transform.ts

@@ -1,3 +1,4 @@
+import * as cdk from "aws-cdk-lib";
 import * as databrew from "aws-cdk-lib/aws-databrew";
 import * as events from "aws-cdk-lib/aws-events";
 import * as iam from "aws-cdk-lib/aws-iam";
@@ -12,9 +13,23 @@ export interface DataBrewTransformStageProps extends StateMachineStageProps {
   readonly jobRoleArn?: string;
   readonly jobType?: string;
   readonly datasetName?: string;
-  readonly recipe?: databrew.CfnJob.RecipeProperty;
-  readonly outputs?: databrew.CfnJob.OutputProperty[];
   readonly createJob?: boolean;
+  readonly dataCatalogOutputs?: databrew.CfnJob.DataCatalogOutputProperty[];
+  readonly databaseOutputs?: databrew.CfnJob.DatabaseOutputProperty[];
+  readonly encryptionKeyArn?: string;
+  readonly encryptionMode?: string;
+  readonly jobSample?: databrew.CfnJob.JobSampleProperty;
+  readonly logSubscription?: string;
+  readonly maxCapacity?: number;
+  readonly maxRetries?: number;
+  readonly outputLocation?: databrew.CfnJob.OutputLocationProperty;
+  readonly outputs?: databrew.CfnJob.OutputProperty[];
+  readonly profileConfiguration?: databrew.CfnJob.ProfileConfigurationProperty;
+  readonly projectName?: string;
+  readonly recipe?: databrew.CfnJob.RecipeProperty;
+  readonly tags?: cdk.CfnTag[];
+  readonly timeout?: number;
+  readonly validationConfigurations?: databrew.CfnJob.ValidationConfigurationProperty[];
 }
 
 export class DataBrewTransformStage extends StateMachineStage {
@@ -28,21 +43,34 @@ export class DataBrewTransformStage extends StateMachineStage {
   constructor(scope: Construct, id: string, props: DataBrewTransformStageProps) {
     super(scope, id, props);
 
-    const { jobName, jobRoleArn, jobType, datasetName, recipe, outputs, createJob } = props;
-    this.jobName = jobName ? jobName : `${id}-job`;
-    this.createJob = jobName && !createJob ? false : true;
+    this.jobName = props.jobName ? props.jobName : `${id}-job`;
+    this.createJob = props.jobName && !props.createJob ? false : true;
 
     if (this.createJob) {
-      if (!jobType) {
+      if (!props.jobType) {
         throw new Error("if 'jobType' is a required property when creating a new DataBrew job");
       }
       this.job = new databrew.CfnJob(this, "DataBrew Job", {
         name: this.jobName,
-        roleArn: jobRoleArn ? jobRoleArn : this.createDefaultDataBrewJobRole().roleArn,
-        type: jobType,
-        datasetName: datasetName,
-        recipe: recipe,
-        outputs: outputs,
+        roleArn: props.jobRoleArn ? props.jobRoleArn : this.createDefaultDataBrewJobRole().roleArn,
+        type: props.jobType,
+        datasetName: props.datasetName,
+        dataCatalogOutputs: props.dataCatalogOutputs,
+        databaseOutputs: props.databaseOutputs,
+        encryptionKeyArn: props.encryptionKeyArn,
+        encryptionMode: props.encryptionMode,
+        jobSample: props.jobSample,
+        logSubscription: props.logSubscription,
+        maxCapacity: props.maxCapacity,
+        maxRetries: props.maxRetries,
+        outputLocation: props.outputLocation,
+        outputs: props.outputs,
+        profileConfiguration: props.profileConfiguration,
+        projectName: props.projectName,
+        recipe: props.recipe,
+        tags: props.tags,
+        timeout: props.timeout,
+        validationConfigurations: props.validationConfigurations,
       });
     }
     const startJobRun = new tasks.GlueDataBrewStartJobRun(this, "Start DataBrew Job", {

src/stages/glue-transform.ts

@@ -13,6 +13,9 @@ export interface GlueTransformStageProps extends StateMachineStageProps {
   readonly jobProps?: glue_alpha.JobProps;
   readonly jobRunArgs?: { [key: string]: any };
   readonly crawlerName?: string;
+  readonly crawlerRole?: string;
+  readonly databaseName?: string;
+  readonly targets?: glue.CfnCrawler.TargetsProperty;
   readonly crawlerProps?: glue.CfnCrawlerProps;
   readonly crawlerAllowFailure?: boolean;
   readonly stateMachineRetryMaxAttempts?: number;
@@ -93,11 +96,22 @@ export class GlueTransformStage extends StateMachineStage {
   }
 
   private getCrawler(props: GlueTransformStageProps): glue.CfnCrawler {
-    if (!props.crawlerProps) {
-      throw TypeError("'crawlerName' or 'crawlerProps' must be set to instantiate this stage");
+    const role = props.crawlerRole ?? props.crawlerProps?.role;
+    if (!role) {
+      throw TypeError("Crawler Role must be set either by 'crawlerRole' or 'crawlerProps.role");
     }
 
-    const crawler = new glue.CfnCrawler(this, "Crawler", props.crawlerProps);
+    const targets = props.targets ?? props.crawlerProps?.targets;
+    if (!targets) {
+      throw TypeError("Crawler Targets must be set either by 'targets' or 'crawlerProps.targets");
+    }
+
+    const crawler = new glue.CfnCrawler(this, "Crawler", {
+      role: role,
+      databaseName: props.databaseName,
+      targets: targets,
+      ...props.crawlerProps,
+    });
     return crawler;
   }
 }