feat(s3-deployment): CloudFront invalidation (#3213)

see #3106

Author: hoegertn

packages/@aws-cdk/aws-cloudfront/lib/distribution.ts

@@ -6,4 +6,9 @@ export interface IDistribution {
    * The domain name of the distribution
    */
   readonly domainName: string;
+
+  /**
+   * The distribution ID for this distribution.
+   */
+  readonly distributionId: string;
 }

packages/@aws-cdk/aws-s3-deployment/README.md

@@ -63,6 +63,32 @@ By default, the contents of the destination bucket will be deleted when the
 changed. You can use the option `retainOnDelete: true` to disable this behavior,
 in which case the contents will be retained.
 
+## CloudFront Invalidation
+
+You can provide a CloudFront distribution and optional paths to invalidate after the bucket deployment finishes.
+
+```ts
+const bucket = new s3.Bucket(this, 'Destination');
+
+const distribution = new cloudfront.CloudFrontWebDistribution(this, 'Distribution', {
+  originConfigs: [
+    {
+      s3OriginSource: {
+        s3BucketSource: bucket
+      },
+      behaviors : [ {isDefaultBehavior: true}]
+    }
+  ]
+});
+
+new s3deploy.BucketDeployment(this, 'DeployWithInvalidation', {
+  source: s3deploy.Source.asset('./website-dist'),
+  destinationBucket: bucket,
+  distribution,
+  distributionPaths: ['/images/*.png'],
+});
+```
+
 ## Notes
 
  * This library uses an AWS CloudFormation custom resource which about 10MiB in

packages/@aws-cdk/aws-s3-deployment/lambda/src/index.py

@@ -6,6 +6,8 @@
 import traceback
 import logging
 import shutil
+import boto3
+from datetime import datetime
 from uuid import uuid4
 
 from botocore.vendored import requests
@@ -14,6 +16,8 @@
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
+cloudfront = boto3.client('cloudfront')
+
 CFN_SUCCESS = "SUCCESS"
 CFN_FAILED = "FAILED"
 
@@ -40,6 +44,16 @@ def cfn_error(message=None):
             dest_bucket_name   = props['DestinationBucketName']
             dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
             retain_on_delete   = props.get('RetainOnDelete', "true") == "true"
+            distribution_id    = props.get('DistributionId', '')
+
+            default_distribution_path = dest_bucket_prefix
+            if not default_distribution_path.endswith("/"):
+                default_distribution_path += "/"
+            if not default_distribution_path.startswith("/"):
+                default_distribution_path = "/" + default_distribution_path
+            default_distribution_path += "*"
+
+            distribution_paths = props.get('DistributionPaths', [default_distribution_path])
         except KeyError as e:
             cfn_error("missing request resource property %s. props: %s" % (str(e), props))
             return
@@ -84,6 +98,9 @@ def cfn_error(message=None):
         if request_type == "Update" or request_type == "Create":
             s3_deploy(s3_source_zip, s3_dest)
 
+        if distribution_id:
+            cloudfront_invalidate(distribution_id, distribution_paths, physical_id)
+
         cfn_send(event, context, CFN_SUCCESS, physicalResourceId=physical_id)
     except KeyError as e:
         cfn_error("invalid request. Missing key %s" % str(e))
@@ -114,6 +131,23 @@ def s3_deploy(s3_source_zip, s3_dest):
     aws_command("s3", "sync", "--delete", contents_dir, s3_dest)
     shutil.rmtree(workdir)
 
+#---------------------------------------------------------------------------------------------------
+# invalidate files in the CloudFront distribution edge caches
+def cloudfront_invalidate(distribution_id, distribution_paths, physical_id):
+    invalidation_resp = cloudfront.create_invalidation(
+        DistributionId=distribution_id,
+        InvalidationBatch={
+            'Paths': {
+                'Quantity': len(distribution_paths),
+                'Items': distribution_paths
+            },
+            'CallerReference': physical_id,
+        })
+    # by default, will wait up to 10 minutes
+    cloudfront.get_waiter('invalidation_completed').wait(
+        DistributionId=distribution_id,
+        Id=invalidation_resp['Invalidation']['Id'])
+
 #---------------------------------------------------------------------------------------------------
 # executes an "aws" cli command
 def aws_command(*args):

packages/@aws-cdk/aws-s3-deployment/lambda/src/requirements.txt

@@ -1,2 +1,2 @@
 awscli==1.16.34
-
+boto3==1.9.177

packages/@aws-cdk/aws-s3-deployment/lambda/test/test.py

@@ -6,13 +6,17 @@
 import sys
 import traceback
 import logging
+import botocore
 from botocore.vendored import requests
+from botocore.exceptions import ClientError
 from unittest.mock import MagicMock
+from unittest.mock import patch
+
 
 class TestHandler(unittest.TestCase):
     def setUp(self):
         logger = logging.getLogger()
-
+        
         # clean up old aws.out file (from previous runs)
         try: os.remove("aws.out")
         except OSError: pass
@@ -133,6 +137,75 @@ def test_update_same_dest(self):
             "s3 sync --delete contents.zip s3://<dest-bucket-name>/"
         )
 
+    def test_update_same_dest_cf_invalidate(self):
+        def mock_make_api_call(self, operation_name, kwarg):
+            if operation_name == 'CreateInvalidation':
+                assert kwarg['DistributionId'] == '<cf-dist-id>'
+                assert kwarg['InvalidationBatch']['Paths']['Quantity'] == 1
+                assert kwarg['InvalidationBatch']['Paths']['Items'][0] == '/*'
+                assert kwarg['InvalidationBatch']['CallerReference'] == '<physical-id>'
+                return {'Invalidation': {'Id': '<invalidation-id>'}}
+            if operation_name == 'GetInvalidation' and kwarg['Id'] == '<invalidation-id>':
+                return {'Invalidation': {'Id': '<invalidation-id>', 'Status': 'Completed'}}
+            raise ClientError({'Error': {'Code': '500', 'Message': 'Unsupported operation'}}, operation_name)
+
+        with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call):
+            invoke_handler("Update", {
+                "SourceBucketName": "<source-bucket>",
+                "SourceObjectKey": "<source-object-key>",
+                "DestinationBucketName": "<dest-bucket-name>",
+                "DistributionId": "<cf-dist-id>"
+            }, old_resource_props={
+                "DestinationBucketName": "<dest-bucket-name>",
+            }, physical_id="<physical-id>")
+
+    def test_update_same_dest_cf_invalidate_custom_prefix(self):
+        def mock_make_api_call(self, operation_name, kwarg):
+            if operation_name == 'CreateInvalidation':
+                assert kwarg['DistributionId'] == '<cf-dist-id>'
+                assert kwarg['InvalidationBatch']['Paths']['Quantity'] == 1
+                assert kwarg['InvalidationBatch']['Paths']['Items'][0] == '/<dest-prefix>/*'
+                assert kwarg['InvalidationBatch']['CallerReference'] == '<physical-id>'
+                return {'Invalidation': {'Id': '<invalidation-id>'}}
+            if operation_name == 'GetInvalidation' and kwarg['Id'] == '<invalidation-id>':
+                return {'Invalidation': {'Id': '<invalidation-id>', 'Status': 'Completed'}}
+            raise ClientError({'Error': {'Code': '500', 'Message': 'Unsupported operation'}}, operation_name)
+
+        with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call):
+            invoke_handler("Update", {
+                "SourceBucketName": "<source-bucket>",
+                "SourceObjectKey": "<source-object-key>",
+                "DestinationBucketName": "<dest-bucket-name>",
+                "DestinationBucketKeyPrefix": "<dest-prefix>",
+                "DistributionId": "<cf-dist-id>"
+            }, old_resource_props={
+                "DestinationBucketName": "<dest-bucket-name>",
+            }, physical_id="<physical-id>")
+
+    def test_update_same_dest_cf_invalidate_custom_paths(self):
+        def mock_make_api_call(self, operation_name, kwarg):
+            if operation_name == 'CreateInvalidation':
+                assert kwarg['DistributionId'] == '<cf-dist-id>'
+                assert kwarg['InvalidationBatch']['Paths']['Quantity'] == 2
+                assert kwarg['InvalidationBatch']['Paths']['Items'][0] == '/path1/*'
+                assert kwarg['InvalidationBatch']['Paths']['Items'][1] == '/path2/*'
+                assert kwarg['InvalidationBatch']['CallerReference'] == '<physical-id>'
+                return {'Invalidation': {'Id': '<invalidation-id>'}}
+            if operation_name == 'GetInvalidation' and kwarg['Id'] == '<invalidation-id>':
+                return {'Invalidation': {'Id': '<invalidation-id>', 'Status': 'Completed'}}
+            raise ClientError({'Error': {'Code': '500', 'Message': 'Unsupported operation'}}, operation_name)
+
+        with patch('botocore.client.BaseClient._make_api_call', new=mock_make_api_call):
+            invoke_handler("Update", {
+                "SourceBucketName": "<source-bucket>",
+                "SourceObjectKey": "<source-object-key>",
+                "DestinationBucketName": "<dest-bucket-name>",
+                "DistributionId": "<cf-dist-id>",
+                "DistributionPaths": ["/path1/*", "/path2/*"]
+            }, old_resource_props={
+                "DestinationBucketName": "<dest-bucket-name>",
+            }, physical_id="<physical-id>")
+
     def test_update_new_dest_retain(self):
         invoke_handler("Update", {
             "SourceBucketName": "<source-bucket>",

packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts

@@ -1,4 +1,6 @@
 import cloudformation = require('@aws-cdk/aws-cloudformation');
+import cloudfront = require('@aws-cdk/aws-cloudfront');
+import iam = require('@aws-cdk/aws-iam');
 import lambda = require('@aws-cdk/aws-lambda');
 import s3 = require('@aws-cdk/aws-s3');
 import cdk = require('@aws-cdk/core');
@@ -37,12 +39,32 @@ export interface BucketDeploymentProps {
    * @default true - when resource is deleted/updated, files are retained
    */
   readonly retainOnDelete?: boolean;
+
+  /**
+   * The CloudFront distribution using the destination bucket as an origin.
+   * Files in the distribution's edge caches will be invalidated after
+   * files are uploaded to the destination bucket.
+   *
+   * @default - No invalidation occurs
+   */
+  readonly distribution?: cloudfront.IDistribution;
+
+  /**
+   * The file paths to invalidate in the CloudFront distribution.
+   *
+   * @default - All files under the destination bucket key prefix will be invalidated.
+   */
+  readonly distributionPaths?: string[];
 }
 
 export class BucketDeployment extends cdk.Construct {
   constructor(scope: cdk.Construct, id: string, props: BucketDeploymentProps) {
     super(scope, id);
 
+    if (props.distributionPaths && !props.distribution) {
+      throw new Error("Distribution must be specified if distribution paths are specified");
+    }
+
     const handler = new lambda.SingletonFunction(this, 'CustomResourceHandler', {
       uuid: '8693BB64-9689-44B6-9AAF-B0CC9EB8756C',
       code: lambda.Code.asset(handlerCodeBundle),
@@ -56,6 +78,13 @@ export class BucketDeployment extends cdk.Construct {
 
     source.bucket.grantRead(handler);
     props.destinationBucket.grantReadWrite(handler);
+    if (props.distribution) {
+      handler.addToRolePolicy(new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: ['cloudfront:GetInvalidation', 'cloudfront:CreateInvalidation'],
+        resources: ['*'],
+      }));
+    }
 
     new cloudformation.CustomResource(this, 'CustomResource', {
       provider: cloudformation.CustomResourceProvider.lambda(handler),
@@ -65,7 +94,9 @@ export class BucketDeployment extends cdk.Construct {
         SourceObjectKey: source.zipObjectKey,
         DestinationBucketName: props.destinationBucket.bucketName,
         DestinationBucketKeyPrefix: props.destinationKeyPrefix,
-        RetainOnDelete: props.retainOnDelete
+        RetainOnDelete: props.retainOnDelete,
+        DistributionId: props.distribution ? props.distribution.distributionId : undefined,
+        DistributionPaths: props.distributionPaths
       }
     });
   }

packages/@aws-cdk/aws-s3-deployment/package.json

@@ -83,6 +83,7 @@
   },
   "dependencies": {
     "@aws-cdk/aws-cloudformation": "^1.3.0",
+    "@aws-cdk/aws-cloudfront": "^1.3.0",
     "@aws-cdk/aws-iam": "^1.3.0",
     "@aws-cdk/aws-lambda": "^1.3.0",
     "@aws-cdk/aws-s3": "^1.3.0",
@@ -92,6 +93,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-cloudformation": "^1.3.0",
+    "@aws-cdk/aws-cloudfront": "^1.3.0",
     "@aws-cdk/aws-iam": "^1.3.0",
     "@aws-cdk/aws-lambda": "^1.3.0",
     "@aws-cdk/aws-s3": "^1.3.0",