fix(aws-ec2): flow log destinationOptions requires all properties (#2…

…1042) PR #20765 introduced destinationOptions, but only introduced one of the optional properties ('hiveCompatiblePartitions') since that is the only property that was relevant for the PR. The [docs](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-flowlog.html#cfn-ec2-flowlog-destinationoptions) don't specify this, but if you provide `destinationOptions` you must specify a value for each prop, otherwise you will receive an error message on deploy. This PR adds the two additional properties. re #21037 ---- ### All Submissions: * [ ] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Jul 7, 2022 · 0a76009 · 0a76009
1 parent 4b4af84
commit 0a76009
Show file tree

Hide file tree

Showing 13 changed files with 492 additions and 63 deletions.
diff --git a/packages/@aws-cdk/aws-ec2/lib/vpc-flow-logs.ts b/packages/@aws-cdk/aws-ec2/lib/vpc-flow-logs.ts
@@ -99,6 +99,23 @@ export abstract class FlowLogResourceType {
   public abstract resourceId: string;
 }
 
+/**
+ * The file format for flow logs written to an S3 bucket destination
+ */
+export enum FlowLogFileFormat {
+  /**
+   * File will be written as plain text
+   *
+   * This is the default value
+   */
+  PLAIN_TEXT = 'plain-text',
+
+  /**
+   * File will be written in parquet format
+   */
+  PARQUET = 'parquet',
+}
+
 /**
  * Options for writing logs to a S3 destination
  */
@@ -110,6 +127,20 @@ export interface S3DestinationOptions {
    * @default false
    */
   readonly hiveCompatiblePartitions?: boolean;
+
+  /**
+   * The format for the flow log
+   *
+   * @default FlowLogFileFormat.PLAIN_TEXT
+   */
+  readonly fileFormat?: FlowLogFileFormat;
+
+  /**
+   * Partition the flow log per hour
+   *
+   * @default false
+   */
+  readonly perHourPartition?: boolean;
 }
 
 /**
@@ -287,7 +318,13 @@ class S3Destination extends FlowLogDestination {
       logDestinationType: FlowLogDestinationType.S3,
       s3Bucket,
       keyPrefix: this.props.keyPrefix,
-      destinationOptions: this.props.destinationOptions,
+      destinationOptions: (this.props.destinationOptions?.fileFormat || this.props.destinationOptions?.perHourPartition
+      || this.props.destinationOptions?.hiveCompatiblePartitions)
+        ? {
+          fileFormat: this.props.destinationOptions.fileFormat ?? FlowLogFileFormat.PLAIN_TEXT,
+          perHourPartition: this.props.destinationOptions.perHourPartition ?? false,
+          hiveCompatiblePartitions: this.props.destinationOptions.hiveCompatiblePartitions ?? false,
+        } : undefined,
     };
   }
 }

diff --git a/packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs.ts b/packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs.ts
@@ -20,6 +20,12 @@ class FeatureFlagStack extends Stack {
     this.bucket = flowLog.bucket!;
     this.bucketArn = this.exportValue(flowLog.bucket!.bucketArn);
 
+    vpc.addFlowLog('FlowLogsS3WithDestinationOptions', {
+      destination: FlowLogDestination.toS3(undefined, undefined, {
+        hiveCompatiblePartitions: true,
+      }),
+    });
+
     new Instance(this, 'FlowLogsInstance', {
       vpc,
       instanceType: InstanceType.of(InstanceClass.T3, InstanceSize.SMALL),

diff --git a/...2/test/vpc-flow-logs.integ.snapshot/FlowLogsDefaultTestDeployAssert6AFD1854.template.json b/...2/test/vpc-flow-logs.integ.snapshot/FlowLogsDefaultTestDeployAssert6AFD1854.template.json
@@ -30,7 +30,7 @@
      }
     },
     "flattenResponse": "false",
-    "salt": "1656511764157"
+    "salt": "1657219152690"
    },
    "UpdateReplacePolicy": "Delete",
    "DeletionPolicy": "Delete"
@@ -51,7 +51,7 @@
      ]
     },
     "expected": "{\"$ObjectLike\":{\"KeyCount\":1}}",
-    "salt": "1656511764158"
+    "salt": "1657219152690"
    },
    "UpdateReplacePolicy": "Delete",
    "DeletionPolicy": "Delete"
@@ -133,7 +133,7 @@
     "Runtime": "nodejs14.x",
     "Code": {
      "S3Bucket": {
-      "Ref": "AssetParameters41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736S3BucketA9F12763"
+      "Ref": "AssetParametersec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3S3Bucket5F1832C4"
      },
      "S3Key": {
       "Fn::Join": [
@@ -146,7 +146,7 @@
            "Fn::Split": [
             "||",
             {
-             "Ref": "AssetParameters41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736S3VersionKey589F30A2"
+             "Ref": "AssetParametersec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3S3VersionKeyA04E23E6"
             }
            ]
           }
@@ -159,7 +159,7 @@
            "Fn::Split": [
             "||",
             {
-             "Ref": "AssetParameters41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736S3VersionKey589F30A2"
+             "Ref": "AssetParametersec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3S3VersionKeyA04E23E6"
             }
            ]
           }
@@ -191,17 +191,17 @@
   }
  },
  "Parameters": {
-  "AssetParameters41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736S3BucketA9F12763": {
+  "AssetParametersec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3S3Bucket5F1832C4": {
    "Type": "String",
-   "Description": "S3 bucket for asset \"41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736\""
+   "Description": "S3 bucket for asset \"ec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3\""
   },
-  "AssetParameters41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736S3VersionKey589F30A2": {
+  "AssetParametersec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3S3VersionKeyA04E23E6": {
    "Type": "String",
-   "Description": "S3 key for asset version \"41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736\""
+   "Description": "S3 key for asset version \"ec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3\""
   },
-  "AssetParameters41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736ArtifactHash2CC614EA": {
+  "AssetParametersec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3ArtifactHash000AF521": {
    "Type": "String",
-   "Description": "Artifact hash for asset \"41fc8f2dc7c01b34dda9916c7f763e7b7909eb629da9ffe879cb786114aae736\""
+   "Description": "Artifact hash for asset \"ec094b96e98289a8faed4f4280a8531224c0191f583bc684c21c91a65319e4a3\""
   }
  }
 }
diff --git a/...ages/@aws-cdk/aws-ec2/test/vpc-flow-logs.integ.snapshot/FlowLogsFeatureFlag.template.json b/...ages/@aws-cdk/aws-ec2/test/vpc-flow-logs.integ.snapshot/FlowLogsFeatureFlag.template.json
@@ -532,6 +532,160 @@
     ]
    }
   },
+  "VPCFlowLogsS3WithDestinationOptionsBucket1B7AC456": {
+   "Type": "AWS::S3::Bucket",
+   "Properties": {
+    "Tags": [
+     {
+      "Key": "Name",
+      "Value": "FlowLogsFeatureFlag/VPC"
+     }
+    ]
+   },
+   "UpdateReplacePolicy": "Retain",
+   "DeletionPolicy": "Retain"
+  },
+  "VPCFlowLogsS3WithDestinationOptionsBucketPolicy35257B71": {
+   "Type": "AWS::S3::BucketPolicy",
+   "Properties": {
+    "Bucket": {
+     "Ref": "VPCFlowLogsS3WithDestinationOptionsBucket1B7AC456"
+    },
+    "PolicyDocument": {
+     "Statement": [
+      {
+       "Action": "s3:PutObject",
+       "Condition": {
+        "StringEquals": {
+         "s3:x-amz-acl": "bucket-owner-full-control",
+         "aws:SourceAccount": {
+          "Ref": "AWS::AccountId"
+         }
+        },
+        "ArnLike": {
+         "aws:SourceArn": {
+          "Fn::Join": [
+           "",
+           [
+            "arn:",
+            {
+             "Ref": "AWS::Partition"
+            },
+            ":logs:",
+            {
+             "Ref": "AWS::Region"
+            },
+            ":",
+            {
+             "Ref": "AWS::AccountId"
+            },
+            ":*"
+           ]
+          ]
+         }
+        }
+       },
+       "Effect": "Allow",
+       "Principal": {
+        "Service": "delivery.logs.amazonaws.com"
+       },
+       "Resource": {
+        "Fn::Join": [
+         "",
+         [
+          {
+           "Fn::GetAtt": [
+            "VPCFlowLogsS3WithDestinationOptionsBucket1B7AC456",
+            "Arn"
+           ]
+          },
+          "/AWSLogs/aws-account-id=",
+          {
+           "Ref": "AWS::AccountId"
+          },
+          "/*"
+         ]
+        ]
+       }
+      },
+      {
+       "Action": [
+        "s3:GetBucketAcl",
+        "s3:ListBucket"
+       ],
+       "Condition": {
+        "StringEquals": {
+         "aws:SourceAccount": {
+          "Ref": "AWS::AccountId"
+         }
+        },
+        "ArnLike": {
+         "aws:SourceArn": {
+          "Fn::Join": [
+           "",
+           [
+            "arn:",
+            {
+             "Ref": "AWS::Partition"
+            },
+            ":logs:",
+            {
+             "Ref": "AWS::Region"
+            },
+            ":",
+            {
+             "Ref": "AWS::AccountId"
+            },
+            ":*"
+           ]
+          ]
+         }
+        }
+       },
+       "Effect": "Allow",
+       "Principal": {
+        "Service": "delivery.logs.amazonaws.com"
+       },
+       "Resource": {
+        "Fn::GetAtt": [
+         "VPCFlowLogsS3WithDestinationOptionsBucket1B7AC456",
+         "Arn"
+        ]
+       }
+      }
+     ],
+     "Version": "2012-10-17"
+    }
+   }
+  },
+  "VPCFlowLogsS3WithDestinationOptionsFlowLog030C15B2": {
+   "Type": "AWS::EC2::FlowLog",
+   "Properties": {
+    "ResourceId": {
+     "Ref": "VPCB9E5F0B4"
+    },
+    "ResourceType": "VPC",
+    "TrafficType": "ALL",
+    "DestinationOptions": {
+     "fileFormat": "plain-text",
+     "perHourPartition": false,
+     "hiveCompatiblePartitions": true
+    },
+    "LogDestination": {
+     "Fn::GetAtt": [
+      "VPCFlowLogsS3WithDestinationOptionsBucket1B7AC456",
+      "Arn"
+     ]
+    },
+    "LogDestinationType": "s3",
+    "Tags": [
+     {
+      "Key": "Name",
+      "Value": "FlowLogsFeatureFlag/VPC"
+     }
+    ]
+   }
+  },
   "FlowLogsInstanceInstanceSecurityGroupF61782E0": {
    "Type": "AWS::EC2::SecurityGroup",
    "Properties": {

diff --git a/packages/@aws-cdk/aws-ec2/test/vpc-flow-logs.integ.snapshot/FlowLogsTestStack.template.json b/packages/@aws-cdk/aws-ec2/test/vpc-flow-logs.integ.snapshot/FlowLogsTestStack.template.json
@@ -878,7 +878,7 @@
    "Properties": {
     "Code": {
      "S3Bucket": {
-      "Ref": "AssetParameters17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccbS3Bucket196AD8D5"
+      "Ref": "AssetParameters60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26S3Bucket180EC6B2"
      },
      "S3Key": {
       "Fn::Join": [
@@ -891,7 +891,7 @@
            "Fn::Split": [
             "||",
             {
-             "Ref": "AssetParameters17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccbS3VersionKey53E5B9FA"
+             "Ref": "AssetParameters60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26S3VersionKeyF1ADAF48"
             }
            ]
           }
@@ -904,7 +904,7 @@
            "Fn::Split": [
             "||",
             {
-             "Ref": "AssetParameters17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccbS3VersionKey53E5B9FA"
+             "Ref": "AssetParameters60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26S3VersionKeyF1ADAF48"
             }
            ]
           }
@@ -943,17 +943,17 @@
   }
  },
  "Parameters": {
-  "AssetParameters17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccbS3Bucket196AD8D5": {
+  "AssetParameters60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26S3Bucket180EC6B2": {
    "Type": "String",
-   "Description": "S3 bucket for asset \"17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccb\""
+   "Description": "S3 bucket for asset \"60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26\""
   },
-  "AssetParameters17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccbS3VersionKey53E5B9FA": {
+  "AssetParameters60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26S3VersionKeyF1ADAF48": {
    "Type": "String",
-   "Description": "S3 key for asset version \"17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccb\""
+   "Description": "S3 key for asset version \"60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26\""
   },
-  "AssetParameters17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccbArtifactHash35F1B2CD": {
+  "AssetParameters60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26ArtifactHashF709D3CB": {
    "Type": "String",
-   "Description": "Artifact hash for asset \"17cb4b37288c269a54418db6e9c7c3763b2d1a82bdc374be4653bd366345eccb\""
+   "Description": "Artifact hash for asset \"60767da3831353fede3cfe92efef10580a600592dec8ccbb06c051e95b9c1b26\""
   }
  }
 }