From b7eee0153febe400ec58a8210dc44f62ce3f6b8c Mon Sep 17 00:00:00 2001 From: danielconde Date: Wed, 28 Aug 2019 22:30:22 +0100 Subject: [PATCH 1/6] initial commit --- __mocks__/aws-sdk.js | 10 +- .../__snapshots__/s3-origin.test.js.snap | 80 ++++++++++++++++ __tests__/s3-origin.test.js | 95 ++++++++++++++++++- lib/getOriginConfig.js | 4 +- lib/index.js | 28 +++++- lib/parseInputOrigins.js | 4 +- 6 files changed, 212 insertions(+), 9 deletions(-) diff --git a/__mocks__/aws-sdk.js b/__mocks__/aws-sdk.js index 16f09f3..00b0cfa 100644 --- a/__mocks__/aws-sdk.js +++ b/__mocks__/aws-sdk.js @@ -19,20 +19,28 @@ const mockGetDistributionConfigPromise = promisifyMock(mockGetDistributionConfig const mockDeleteDistribution = jest.fn() const mockDeleteDistributionPromise = promisifyMock(mockDeleteDistribution) +const mockCreateCloudFrontOriginAccessIdentity = jest.fn() +const mockCreateCloudFrontOriginAccessIdentityPromise = promisifyMock( + mockCreateCloudFrontOriginAccessIdentity +) + module.exports = { mockCreateDistribution, mockUpdateDistribution, mockGetDistributionConfig, mockDeleteDistribution, + mockCreateCloudFrontOriginAccessIdentity, mockCreateDistributionPromise, mockUpdateDistributionPromise, mockGetDistributionConfigPromise, mockDeleteDistributionPromise, + mockCreateCloudFrontOriginAccessIdentityPromise, CloudFront: jest.fn(() => ({ createDistribution: mockCreateDistribution, updateDistribution: mockUpdateDistribution, getDistributionConfig: mockGetDistributionConfig, - deleteDistribution: mockDeleteDistribution + deleteDistribution: mockDeleteDistribution, + createCloudFrontOriginAccessIdentity: mockCreateCloudFrontOriginAccessIdentity })) } diff --git a/__tests__/__snapshots__/s3-origin.test.js.snap b/__tests__/__snapshots__/s3-origin.test.js.snap index 5b4658f..52c5dc2 100644 --- a/__tests__/__snapshots__/s3-origin.test.js.snap +++ b/__tests__/__snapshots__/s3-origin.test.js.snap @@ -1,5 +1,85 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Input origin as an S3 bucket url creates distribution configured to serve private S3 content 1`] = ` +Object { + "DistributionConfig": Object { + "Aliases": Object { + "Items": Array [], + "Quantity": 0, + }, + "CallerReference": "1566599541192", + "Comment": "", + "DefaultCacheBehavior": Object { + "AllowedMethods": Object { + "CachedMethods": Object { + "Items": Array [ + "HEAD", + "GET", + ], + "Quantity": 2, + }, + "Items": Array [ + "HEAD", + "GET", + ], + "Quantity": 2, + }, + "Compress": false, + "DefaultTTL": 86400, + "FieldLevelEncryptionId": "", + "ForwardedValues": Object { + "Cookies": Object { + "Forward": "none", + }, + "Headers": Object { + "Items": Array [], + "Quantity": 0, + }, + "QueryString": false, + "QueryStringCacheKeys": Object { + "Items": Array [], + "Quantity": 0, + }, + }, + "LambdaFunctionAssociations": Object { + "Items": Array [], + "Quantity": 0, + }, + "MaxTTL": 31536000, + "MinTTL": 0, + "SmoothStreaming": false, + "TargetOriginId": "mybucket", + "TrustedSigners": Object { + "Enabled": false, + "Items": Array [], + "Quantity": 0, + }, + "ViewerProtocolPolicy": "redirect-to-https", + }, + "Enabled": true, + "HttpVersion": "http2", + "Origins": Object { + "Items": Array [ + Object { + "CustomHeaders": Object { + "Items": Array [], + "Quantity": 0, + }, + "DomainName": "mybucket.s3.amazonaws.com", + "Id": "mybucket", + "OriginPath": "", + "S3OriginConfig": Object { + "OriginAccessIdentity": "s3-canonical-user-id-xyz", + }, + }, + ], + "Quantity": 1, + }, + "PriceClass": "PriceClass_All", + }, +} +`; + exports[`Input origin as an S3 bucket url creates distribution with S3 origin 1`] = ` Object { "DistributionConfig": Object { diff --git a/__tests__/s3-origin.test.js b/__tests__/s3-origin.test.js index 8d392df..d0d2d5a 100644 --- a/__tests__/s3-origin.test.js +++ b/__tests__/s3-origin.test.js @@ -3,7 +3,8 @@ const { mockUpdateDistribution, mockCreateDistributionPromise, mockGetDistributionConfigPromise, - mockUpdateDistributionPromise + mockUpdateDistributionPromise, + mockCreateCloudFrontOriginAccessIdentityPromise } = require('aws-sdk') const { createComponent } = require('../test-utils') @@ -51,6 +52,98 @@ describe('Input origin as an S3 bucket url', () => { expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() }) + it('creates distribution configured to serve private S3 content', async () => { + mockCreateCloudFrontOriginAccessIdentityPromise.mockResolvedValueOnce({ + CloudFrontOriginAccessIdentity: { + Id: 'access-identity-xyz', + S3CanonicalUserId: 's3-canonical-user-id-xyz' + } + }) + + await component.default({ + origins: [ + { + url: 'https://mybucket.s3.amazonaws.com', + private: true + } + ] + }) + + expect(mockCreateDistribution).toBeCalledWith( + expect.objectContaining({ + DistributionConfig: expect.objectContaining({ + Origins: expect.objectContaining({ + Items: [ + { + Id: 'mybucket', + DomainName: 'mybucket.s3.amazonaws.com', + S3OriginConfig: { + OriginAccessIdentity: 's3-canonical-user-id-xyz' + }, + CustomHeaders: { + Quantity: 0, + Items: [] + }, + OriginPath: '' + } + ] + }) + }) + }) + ) + expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() + }) + + it.skip('updates distribution configured to serve private S3 content', async () => { + mockGetDistributionConfigPromise.mockResolvedValueOnce({ + ETag: 'etag', + DistributionConfig: { + Origins: { + Items: [ + { + S3OriginConfig: { + OriginAccessIdentity: 's3-canonical-user-id-xyz' + } + } + ] + } + } + }) + + await component.default({ + origins: [ + { + url: 'https://mybucket.s3.amazonaws.com', + private: true + } + ] + }) + + expect(mockUpdateDistribution).toBeCalledWith( + expect.objectContaining({ + DistributionConfig: expect.objectContaining({ + Origins: expect.objectContaining({ + Items: [ + { + Id: 'mybucket', + DomainName: 'mybucket.s3.amazonaws.com', + S3OriginConfig: { + OriginAccessIdentity: 's3-canonical-user-id-xyz' + }, + CustomHeaders: { + Quantity: 0, + Items: [] + }, + OriginPath: '' + } + ] + }) + }) + }) + ) + expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() + }) + it('updates distribution', async () => { mockGetDistributionConfigPromise.mockResolvedValueOnce({ ETag: 'etag', diff --git a/lib/getOriginConfig.js b/lib/getOriginConfig.js index a7fe6c4..a58bb06 100644 --- a/lib/getOriginConfig.js +++ b/lib/getOriginConfig.js @@ -1,6 +1,6 @@ const url = require('url') -module.exports = (origin) => { +module.exports = (origin, { s3CanonicalUserId = '' }) => { const originUrl = typeof origin === 'string' ? origin : origin.url const { hostname } = url.parse(originUrl) @@ -20,7 +20,7 @@ module.exports = (origin) => { originConfig.Id = bucketName originConfig.DomainName = `${bucketName}.s3.amazonaws.com` originConfig.S3OriginConfig = { - OriginAccessIdentity: '' + OriginAccessIdentity: s3CanonicalUserId } } else { originConfig.CustomOriginConfig = { diff --git a/lib/index.js b/lib/index.js index b883bd2..f2390eb 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,7 +1,12 @@ const parseInputOrigins = require('./parseInputOrigins') const getDefaultCacheBehavior = require('./getDefaultCacheBehavior') -const createCloudFrontDistribution = async (cf, inputs) => { +const servePrivateContentEnabled = (inputs) => + inputs.origins.some((origin) => { + return origin && origin.private === true + }) + +const createCloudFrontDistribution = async (cf, inputs, options) => { const params = { DistributionConfig: { CallerReference: String(Date.now()), @@ -22,7 +27,24 @@ const createCloudFrontDistribution = async (cf, inputs) => { const distributionConfig = params.DistributionConfig - const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins) + let s3CanonicalUserId + + if (servePrivateContentEnabled(inputs)) { + const { + CloudFrontOriginAccessIdentity: { S3CanonicalUserId } + } = await cf + .createCloudFrontOriginAccessIdentity({ + CloudFrontOriginAccessIdentityConfig: { + CallerReference: 'serverless-managed-cloudfront-access-identity', + Comment: 'CloudFront Origin Access Identity created to allow serving private S3 content' + } + }) + .promise() + + s3CanonicalUserId = S3CanonicalUserId + } + + const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, { s3CanonicalUserId }) distributionConfig.Origins = Origins @@ -63,7 +85,7 @@ const updateCloudFrontDistribution = async (cf, distributionId, inputs) => { params.DistributionConfig.Enabled = inputs.enabled === false ? false : true - const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins) + const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, {}) params.DistributionConfig.DefaultCacheBehavior = getDefaultCacheBehavior(Origins.Items[0].Id) params.DistributionConfig.Origins = Origins diff --git a/lib/parseInputOrigins.js b/lib/parseInputOrigins.js index 51b9c2c..93d9577 100644 --- a/lib/parseInputOrigins.js +++ b/lib/parseInputOrigins.js @@ -1,7 +1,7 @@ const getOriginConfig = require('./getOriginConfig') const getCacheBehavior = require('./getCacheBehavior') -module.exports = (origins) => { +module.exports = (origins, options) => { const distributionOrigins = { Quantity: 0, Items: [] @@ -9,7 +9,7 @@ module.exports = (origins) => { let distributionCacheBehaviors for (const origin of origins) { - const originConfig = getOriginConfig(origin) + const originConfig = getOriginConfig(origin, options) distributionOrigins.Quantity = distributionOrigins.Quantity + 1 distributionOrigins.Items.push(originConfig) From 18902849dc9957131ac3371dcf75af1e2a1cc571 Mon Sep 17 00:00:00 2001 From: danielconde Date: Thu, 29 Aug 2019 21:10:53 +0100 Subject: [PATCH 2/6] create origin access identity on update too --- .../__snapshots__/s3-origin.test.js.snap | 82 ++++++++++++++++++- __tests__/s3-origin.test.js | 47 +++++++---- lib/createOriginAccessIdentity.js | 16 ++++ lib/getOriginConfig.js | 6 +- lib/index.js | 31 ++++--- 5 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 lib/createOriginAccessIdentity.js diff --git a/__tests__/__snapshots__/s3-origin.test.js.snap b/__tests__/__snapshots__/s3-origin.test.js.snap index 52c5dc2..e3edbca 100644 --- a/__tests__/__snapshots__/s3-origin.test.js.snap +++ b/__tests__/__snapshots__/s3-origin.test.js.snap @@ -69,7 +69,7 @@ Object { "Id": "mybucket", "OriginPath": "", "S3OriginConfig": Object { - "OriginAccessIdentity": "s3-canonical-user-id-xyz", + "OriginAccessIdentity": "origin-access-identity/cloudfront/access-identity-xyz", }, }, ], @@ -233,3 +233,83 @@ Object { "IfMatch": "etag", } `; + +exports[`Input origin as an S3 bucket url updates distribution configured to serve private S3 content 1`] = ` +Object { + "DistributionConfig": Object { + "Aliases": Object { + "Items": Array [], + "Quantity": 0, + }, + "CallerReference": "1566599541192", + "Comment": "", + "DefaultCacheBehavior": Object { + "AllowedMethods": Object { + "CachedMethods": Object { + "Items": Array [ + "HEAD", + "GET", + ], + "Quantity": 2, + }, + "Items": Array [ + "HEAD", + "GET", + ], + "Quantity": 2, + }, + "Compress": false, + "DefaultTTL": 86400, + "FieldLevelEncryptionId": "", + "ForwardedValues": Object { + "Cookies": Object { + "Forward": "none", + }, + "Headers": Object { + "Items": Array [], + "Quantity": 0, + }, + "QueryString": false, + "QueryStringCacheKeys": Object { + "Items": Array [], + "Quantity": 0, + }, + }, + "LambdaFunctionAssociations": Object { + "Items": Array [], + "Quantity": 0, + }, + "MaxTTL": 31536000, + "MinTTL": 0, + "SmoothStreaming": false, + "TargetOriginId": "mybucket", + "TrustedSigners": Object { + "Enabled": false, + "Items": Array [], + "Quantity": 0, + }, + "ViewerProtocolPolicy": "redirect-to-https", + }, + "Enabled": true, + "HttpVersion": "http2", + "Origins": Object { + "Items": Array [ + Object { + "CustomHeaders": Object { + "Items": Array [], + "Quantity": 0, + }, + "DomainName": "mybucket.s3.amazonaws.com", + "Id": "mybucket", + "OriginPath": "", + "S3OriginConfig": Object { + "OriginAccessIdentity": "origin-access-identity/cloudfront/access-identity-xyz", + }, + }, + ], + "Quantity": 1, + }, + "PriceClass": "PriceClass_All", + }, +} +`; diff --git a/__tests__/s3-origin.test.js b/__tests__/s3-origin.test.js index d0d2d5a..d9c8990 100644 --- a/__tests__/s3-origin.test.js +++ b/__tests__/s3-origin.test.js @@ -78,7 +78,7 @@ describe('Input origin as an S3 bucket url', () => { Id: 'mybucket', DomainName: 'mybucket.s3.amazonaws.com', S3OriginConfig: { - OriginAccessIdentity: 's3-canonical-user-id-xyz' + OriginAccessIdentity: 'origin-access-identity/cloudfront/access-identity-xyz' }, CustomHeaders: { Quantity: 0, @@ -94,22 +94,29 @@ describe('Input origin as an S3 bucket url', () => { expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() }) - it.skip('updates distribution configured to serve private S3 content', async () => { + it('updates distribution configured to serve private S3 content', async () => { + mockCreateCloudFrontOriginAccessIdentityPromise.mockResolvedValue({ + CloudFrontOriginAccessIdentity: { + Id: 'access-identity-xyz', + S3CanonicalUserId: 's3-canonical-user-id-xyz' + } + }) + mockGetDistributionConfigPromise.mockResolvedValueOnce({ ETag: 'etag', DistributionConfig: { Origins: { - Items: [ - { - S3OriginConfig: { - OriginAccessIdentity: 's3-canonical-user-id-xyz' - } - } - ] + Items: [] } } }) + mockUpdateDistributionPromise.mockResolvedValueOnce({ + Distribution: { + Id: 'distributionwithS3originupdated' + } + }) + await component.default({ origins: [ { @@ -119,22 +126,28 @@ describe('Input origin as an S3 bucket url', () => { ] }) + await component.default({ + origins: [ + { + url: 'https://anotherbucket.s3.amazonaws.com', + private: true + } + ] + }) + expect(mockUpdateDistribution).toBeCalledWith( expect.objectContaining({ DistributionConfig: expect.objectContaining({ Origins: expect.objectContaining({ Items: [ { - Id: 'mybucket', - DomainName: 'mybucket.s3.amazonaws.com', + Id: 'anotherbucket', + DomainName: 'anotherbucket.s3.amazonaws.com', S3OriginConfig: { - OriginAccessIdentity: 's3-canonical-user-id-xyz' + OriginAccessIdentity: 'origin-access-identity/cloudfront/access-identity-xyz' }, - CustomHeaders: { - Quantity: 0, - Items: [] - }, - OriginPath: '' + OriginPath: '', + CustomHeaders: { Items: [], Quantity: 0 } } ] }) diff --git a/lib/createOriginAccessIdentity.js b/lib/createOriginAccessIdentity.js new file mode 100644 index 0000000..69892a0 --- /dev/null +++ b/lib/createOriginAccessIdentity.js @@ -0,0 +1,16 @@ +module.exports = async (cf) => { + const { + CloudFrontOriginAccessIdentity: { Id } + } = await cf + .createCloudFrontOriginAccessIdentity({ + CloudFrontOriginAccessIdentityConfig: { + CallerReference: 'serverless-managed-cloudfront-access-identity', + Comment: 'CloudFront Origin Access Identity created to allow serving private S3 content' + } + }) + .promise() + + console.log('TCL: S3CanonicalUserId', Id) + + return Id +} diff --git a/lib/getOriginConfig.js b/lib/getOriginConfig.js index a58bb06..16b15f4 100644 --- a/lib/getOriginConfig.js +++ b/lib/getOriginConfig.js @@ -1,6 +1,6 @@ const url = require('url') -module.exports = (origin, { s3CanonicalUserId = '' }) => { +module.exports = (origin, { originAccessIdentityId = '' }) => { const originUrl = typeof origin === 'string' ? origin : origin.url const { hostname } = url.parse(originUrl) @@ -20,7 +20,9 @@ module.exports = (origin, { s3CanonicalUserId = '' }) => { originConfig.Id = bucketName originConfig.DomainName = `${bucketName}.s3.amazonaws.com` originConfig.S3OriginConfig = { - OriginAccessIdentity: s3CanonicalUserId + OriginAccessIdentity: originAccessIdentityId + ? `origin-access-identity/cloudfront/${originAccessIdentityId}` + : '' } } else { originConfig.CustomOriginConfig = { diff --git a/lib/index.js b/lib/index.js index f2390eb..430cbd2 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,12 +1,13 @@ const parseInputOrigins = require('./parseInputOrigins') const getDefaultCacheBehavior = require('./getDefaultCacheBehavior') +const createOriginAccessIdentity = require('./createOriginAccessIdentity') const servePrivateContentEnabled = (inputs) => inputs.origins.some((origin) => { return origin && origin.private === true }) -const createCloudFrontDistribution = async (cf, inputs, options) => { +const createCloudFrontDistribution = async (cf, inputs) => { const params = { DistributionConfig: { CallerReference: String(Date.now()), @@ -27,24 +28,13 @@ const createCloudFrontDistribution = async (cf, inputs, options) => { const distributionConfig = params.DistributionConfig - let s3CanonicalUserId + let originAccessIdentityId if (servePrivateContentEnabled(inputs)) { - const { - CloudFrontOriginAccessIdentity: { S3CanonicalUserId } - } = await cf - .createCloudFrontOriginAccessIdentity({ - CloudFrontOriginAccessIdentityConfig: { - CallerReference: 'serverless-managed-cloudfront-access-identity', - Comment: 'CloudFront Origin Access Identity created to allow serving private S3 content' - } - }) - .promise() - - s3CanonicalUserId = S3CanonicalUserId + originAccessIdentityId = await createOriginAccessIdentity(cf) } - const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, { s3CanonicalUserId }) + const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, { originAccessIdentityId }) distributionConfig.Origins = Origins @@ -85,7 +75,16 @@ const updateCloudFrontDistribution = async (cf, distributionId, inputs) => { params.DistributionConfig.Enabled = inputs.enabled === false ? false : true - const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, {}) + let originAccessIdentityId + + if (servePrivateContentEnabled(inputs)) { + // presumably it's ok to call create origin access identity again + // aws api returns cached copy of what was previously created + // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CloudFront.html#createCloudFrontOriginAccessIdentity-property + originAccessIdentityId = await createOriginAccessIdentity(cf) + } + + const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, { originAccessIdentityId }) params.DistributionConfig.DefaultCacheBehavior = getDefaultCacheBehavior(Origins.Items[0].Id) params.DistributionConfig.Origins = Origins From eb495bb45c52804e825515718a359f68e6e2d89d Mon Sep 17 00:00:00 2001 From: danielconde Date: Thu, 29 Aug 2019 22:40:43 +0100 Subject: [PATCH 3/6] update bucket policy with cloudfront access --- __mocks__/aws-sdk.js | 10 + .../__snapshots__/s3-origin.test.js.snap | 44 +-- __tests__/s3-origin.test.js | 265 +++++++++--------- lib/createOriginAccessIdentity.js | 6 +- lib/grantCloudFrontBucketAccess.js | 24 ++ lib/index.js | 30 +- serverless.js | 9 +- 7 files changed, 222 insertions(+), 166 deletions(-) create mode 100644 lib/grantCloudFrontBucketAccess.js diff --git a/__mocks__/aws-sdk.js b/__mocks__/aws-sdk.js index 00b0cfa..37f3cf4 100644 --- a/__mocks__/aws-sdk.js +++ b/__mocks__/aws-sdk.js @@ -24,12 +24,18 @@ const mockCreateCloudFrontOriginAccessIdentityPromise = promisifyMock( mockCreateCloudFrontOriginAccessIdentity ) +const mockPutBucketPolicy = jest.fn() +const mockPutBucketPolicyPromise = promisifyMock(mockPutBucketPolicy) + module.exports = { mockCreateDistribution, mockUpdateDistribution, mockGetDistributionConfig, mockDeleteDistribution, mockCreateCloudFrontOriginAccessIdentity, + mockPutBucketPolicy, + + mockPutBucketPolicyPromise, mockCreateDistributionPromise, mockUpdateDistributionPromise, mockGetDistributionConfigPromise, @@ -42,5 +48,9 @@ module.exports = { getDistributionConfig: mockGetDistributionConfig, deleteDistribution: mockDeleteDistribution, createCloudFrontOriginAccessIdentity: mockCreateCloudFrontOriginAccessIdentity + })), + + S3: jest.fn(() => ({ + putBucketPolicy: mockPutBucketPolicy })) } diff --git a/__tests__/__snapshots__/s3-origin.test.js.snap b/__tests__/__snapshots__/s3-origin.test.js.snap index e3edbca..1e8e9fe 100644 --- a/__tests__/__snapshots__/s3-origin.test.js.snap +++ b/__tests__/__snapshots__/s3-origin.test.js.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Input origin as an S3 bucket url creates distribution configured to serve private S3 content 1`] = ` +exports[`S3 origins When origin is an S3 URL only accessible via CloudFront creates distribution 1`] = ` Object { "DistributionConfig": Object { "Aliases": Object { @@ -80,7 +80,7 @@ Object { } `; -exports[`Input origin as an S3 bucket url creates distribution with S3 origin 1`] = ` +exports[`S3 origins When origin is an S3 URL only accessible via CloudFront updates distribution 1`] = ` Object { "DistributionConfig": Object { "Aliases": Object { @@ -149,7 +149,7 @@ Object { "Id": "mybucket", "OriginPath": "", "S3OriginConfig": Object { - "OriginAccessIdentity": "", + "OriginAccessIdentity": "origin-access-identity/cloudfront/access-identity-xyz", }, }, ], @@ -160,9 +160,15 @@ Object { } `; -exports[`Input origin as an S3 bucket url updates distribution 1`] = ` +exports[`S3 origins When origin is an S3 bucket URL creates distribution 1`] = ` Object { "DistributionConfig": Object { + "Aliases": Object { + "Items": Array [], + "Quantity": 0, + }, + "CallerReference": "1566599541192", + "Comment": "", "DefaultCacheBehavior": Object { "AllowedMethods": Object { "CachedMethods": Object { @@ -202,7 +208,7 @@ Object { "MaxTTL": 31536000, "MinTTL": 0, "SmoothStreaming": false, - "TargetOriginId": "anotherbucket", + "TargetOriginId": "mybucket", "TrustedSigners": Object { "Enabled": false, "Items": Array [], @@ -211,6 +217,7 @@ Object { "ViewerProtocolPolicy": "redirect-to-https", }, "Enabled": true, + "HttpVersion": "http2", "Origins": Object { "Items": Array [ Object { @@ -218,8 +225,8 @@ Object { "Items": Array [], "Quantity": 0, }, - "DomainName": "anotherbucket.s3.amazonaws.com", - "Id": "anotherbucket", + "DomainName": "mybucket.s3.amazonaws.com", + "Id": "mybucket", "OriginPath": "", "S3OriginConfig": Object { "OriginAccessIdentity": "", @@ -228,21 +235,14 @@ Object { ], "Quantity": 1, }, + "PriceClass": "PriceClass_All", }, - "Id": "distributionwithS3origin", - "IfMatch": "etag", } `; -exports[`Input origin as an S3 bucket url updates distribution configured to serve private S3 content 1`] = ` +exports[`S3 origins When origin is an S3 bucket URL updates distribution 1`] = ` Object { "DistributionConfig": Object { - "Aliases": Object { - "Items": Array [], - "Quantity": 0, - }, - "CallerReference": "1566599541192", - "Comment": "", "DefaultCacheBehavior": Object { "AllowedMethods": Object { "CachedMethods": Object { @@ -282,7 +282,7 @@ Object { "MaxTTL": 31536000, "MinTTL": 0, "SmoothStreaming": false, - "TargetOriginId": "mybucket", + "TargetOriginId": "anotherbucket", "TrustedSigners": Object { "Enabled": false, "Items": Array [], @@ -291,7 +291,6 @@ Object { "ViewerProtocolPolicy": "redirect-to-https", }, "Enabled": true, - "HttpVersion": "http2", "Origins": Object { "Items": Array [ Object { @@ -299,17 +298,18 @@ Object { "Items": Array [], "Quantity": 0, }, - "DomainName": "mybucket.s3.amazonaws.com", - "Id": "mybucket", + "DomainName": "anotherbucket.s3.amazonaws.com", + "Id": "anotherbucket", "OriginPath": "", "S3OriginConfig": Object { - "OriginAccessIdentity": "origin-access-identity/cloudfront/access-identity-xyz", + "OriginAccessIdentity": "", }, }, ], "Quantity": 1, }, - "PriceClass": "PriceClass_All", }, + "Id": "distributionwithS3origin", + "IfMatch": "etag", } `; diff --git a/__tests__/s3-origin.test.js b/__tests__/s3-origin.test.js index d21b495..a5ef639 100644 --- a/__tests__/s3-origin.test.js +++ b/__tests__/s3-origin.test.js @@ -4,12 +4,13 @@ const { mockCreateDistributionPromise, mockGetDistributionConfigPromise, mockUpdateDistributionPromise, - mockCreateCloudFrontOriginAccessIdentityPromise + mockCreateCloudFrontOriginAccessIdentityPromise, + mockPutBucketPolicy } = require('aws-sdk') const { createComponent, assertHasOrigin } = require('../test-utils') -describe('Input origin as an S3 bucket url', () => { +describe('S3 origins', () => { let component beforeEach(async () => { @@ -22,160 +23,156 @@ describe('Input origin as an S3 bucket url', () => { component = await createComponent() }) - it('creates distribution with S3 origin', async () => { - await component.default({ - origins: ['https://mybucket.s3.amazonaws.com'] - }) + describe('When origin is an S3 bucket URL', () => { + it('creates distribution', async () => { + await component.default({ + origins: ['https://mybucket.s3.amazonaws.com'] + }) - assertHasOrigin(mockCreateDistribution, { - Id: 'mybucket', - DomainName: 'mybucket.s3.amazonaws.com', - S3OriginConfig: { - OriginAccessIdentity: '' - }, - CustomHeaders: { - Quantity: 0, - Items: [] - }, - OriginPath: '' + assertHasOrigin(mockCreateDistribution, { + Id: 'mybucket', + DomainName: 'mybucket.s3.amazonaws.com', + S3OriginConfig: { + OriginAccessIdentity: '' + }, + CustomHeaders: { + Quantity: 0, + Items: [] + }, + OriginPath: '' + }) + + expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() }) - expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() - }) + it('updates distribution', async () => { + mockGetDistributionConfigPromise.mockResolvedValueOnce({ + ETag: 'etag', + DistributionConfig: { + Origins: { + Items: [] + } + } + }) + mockUpdateDistributionPromise.mockResolvedValueOnce({ + Distribution: { + Id: 'distributionwithS3originupdated' + } + }) - it('creates distribution configured to serve private S3 content', async () => { - mockCreateCloudFrontOriginAccessIdentityPromise.mockResolvedValueOnce({ - CloudFrontOriginAccessIdentity: { - Id: 'access-identity-xyz', - S3CanonicalUserId: 's3-canonical-user-id-xyz' - } + await component.default({ + origins: ['https://mybucket.s3.amazonaws.com'] + }) + + await component.default({ + origins: ['https://anotherbucket.s3.amazonaws.com'] + }) + + assertHasOrigin(mockUpdateDistribution, { + Id: 'anotherbucket', + DomainName: 'anotherbucket.s3.amazonaws.com' + }) + + expect(mockUpdateDistribution.mock.calls[0][0]).toMatchSnapshot() }) + }) - await component.default({ - origins: [ - { - url: 'https://mybucket.s3.amazonaws.com', - private: true + describe('When origin is an S3 URL only accessible via CloudFront', () => { + it('creates distribution', async () => { + mockCreateCloudFrontOriginAccessIdentityPromise.mockResolvedValueOnce({ + CloudFrontOriginAccessIdentity: { + Id: 'access-identity-xyz', + S3CanonicalUserId: 's3-canonical-user-id-xyz' } - ] - }) + }) - expect(mockCreateDistribution).toBeCalledWith( - expect.objectContaining({ - DistributionConfig: expect.objectContaining({ - Origins: expect.objectContaining({ - Items: [ - { - Id: 'mybucket', - DomainName: 'mybucket.s3.amazonaws.com', - S3OriginConfig: { - OriginAccessIdentity: 'origin-access-identity/cloudfront/access-identity-xyz' - }, - CustomHeaders: { - Quantity: 0, - Items: [] - }, - OriginPath: '' - } - ] - }) - }) - }) - ) - expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() - }) + await component.default({ + origins: [ + { + url: 'https://mybucket.s3.amazonaws.com', + private: true + } + ] + }) - it('updates distribution configured to serve private S3 content', async () => { - mockCreateCloudFrontOriginAccessIdentityPromise.mockResolvedValue({ - CloudFrontOriginAccessIdentity: { - Id: 'access-identity-xyz', - S3CanonicalUserId: 's3-canonical-user-id-xyz' - } - }) + expect(mockPutBucketPolicy).toBeCalledWith({ + Bucket: 'mybucket', + Policy: expect.stringContaining('"CanonicalUser":"s3-canonical-user-id-xyz"') + }) - mockGetDistributionConfigPromise.mockResolvedValueOnce({ - ETag: 'etag', - DistributionConfig: { - Origins: { + assertHasOrigin(mockCreateDistribution, { + Id: 'mybucket', + DomainName: 'mybucket.s3.amazonaws.com', + S3OriginConfig: { + OriginAccessIdentity: 'origin-access-identity/cloudfront/access-identity-xyz' + }, + CustomHeaders: { + Quantity: 0, Items: [] - } - } - }) + }, + OriginPath: '' + }) - mockUpdateDistributionPromise.mockResolvedValueOnce({ - Distribution: { - Id: 'distributionwithS3originupdated' - } + expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() }) - await component.default({ - origins: [ - { - url: 'https://mybucket.s3.amazonaws.com', - private: true + it('updates distribution', async () => { + mockCreateCloudFrontOriginAccessIdentityPromise.mockResolvedValue({ + CloudFrontOriginAccessIdentity: { + Id: 'access-identity-xyz', + S3CanonicalUserId: 's3-canonical-user-id-xyz' } - ] - }) + }) - await component.default({ - origins: [ - { - url: 'https://anotherbucket.s3.amazonaws.com', - private: true + mockGetDistributionConfigPromise.mockResolvedValueOnce({ + ETag: 'etag', + DistributionConfig: { + Origins: { + Items: [] + } } - ] - }) - - expect(mockUpdateDistribution).toBeCalledWith( - expect.objectContaining({ - DistributionConfig: expect.objectContaining({ - Origins: expect.objectContaining({ - Items: [ - { - Id: 'anotherbucket', - DomainName: 'anotherbucket.s3.amazonaws.com', - S3OriginConfig: { - OriginAccessIdentity: 'origin-access-identity/cloudfront/access-identity-xyz' - }, - OriginPath: '', - CustomHeaders: { Items: [], Quantity: 0 } - } - ] - }) - }) - }) - ) - expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() - }) + }) - it('updates distribution', async () => { - mockGetDistributionConfigPromise.mockResolvedValueOnce({ - ETag: 'etag', - DistributionConfig: { - Origins: { - Items: [] + mockUpdateDistributionPromise.mockResolvedValueOnce({ + Distribution: { + Id: 'distributionwithS3originupdated' } - } - }) - mockUpdateDistributionPromise.mockResolvedValueOnce({ - Distribution: { - Id: 'distributionwithS3originupdated' - } - }) + }) - await component.default({ - origins: ['https://mybucket.s3.amazonaws.com'] - }) + await component.default({ + origins: [ + { + url: 'https://mybucket.s3.amazonaws.com', + private: true + } + ] + }) - await component.default({ - origins: ['https://anotherbucket.s3.amazonaws.com'] - }) + await component.default({ + origins: [ + { + url: 'https://anotherbucket.s3.amazonaws.com', + private: true + } + ] + }) - assertHasOrigin(mockUpdateDistribution, { - Id: 'anotherbucket', - DomainName: 'anotherbucket.s3.amazonaws.com' - }) + expect(mockPutBucketPolicy).toBeCalledWith({ + Bucket: 'anotherbucket', + Policy: expect.stringContaining('"CanonicalUser":"s3-canonical-user-id-xyz"') + }) - expect(mockUpdateDistribution.mock.calls[0][0]).toMatchSnapshot() + assertHasOrigin(mockUpdateDistribution, { + Id: 'anotherbucket', + DomainName: 'anotherbucket.s3.amazonaws.com', + S3OriginConfig: { + OriginAccessIdentity: 'origin-access-identity/cloudfront/access-identity-xyz' + }, + OriginPath: '', + CustomHeaders: { Items: [], Quantity: 0 } + }) + + expect(mockCreateDistribution.mock.calls[0][0]).toMatchSnapshot() + }) }) }) diff --git a/lib/createOriginAccessIdentity.js b/lib/createOriginAccessIdentity.js index 69892a0..7913fc7 100644 --- a/lib/createOriginAccessIdentity.js +++ b/lib/createOriginAccessIdentity.js @@ -1,6 +1,6 @@ module.exports = async (cf) => { const { - CloudFrontOriginAccessIdentity: { Id } + CloudFrontOriginAccessIdentity: { Id, S3CanonicalUserId } } = await cf .createCloudFrontOriginAccessIdentity({ CloudFrontOriginAccessIdentityConfig: { @@ -10,7 +10,5 @@ module.exports = async (cf) => { }) .promise() - console.log('TCL: S3CanonicalUserId', Id) - - return Id + return { originAccessIdentityId: Id, s3CanonicalUserId: S3CanonicalUserId } } diff --git a/lib/grantCloudFrontBucketAccess.js b/lib/grantCloudFrontBucketAccess.js new file mode 100644 index 0000000..7cd93fb --- /dev/null +++ b/lib/grantCloudFrontBucketAccess.js @@ -0,0 +1,24 @@ +module.exports = (s3, bucketName, s3CanonicalUserId) => { + const policy = ` + { + "Version":"2012-10-17", + "Id":"PolicyForCloudFrontPrivateContent", + "Statement":[ + { + "Sid":" Grant a CloudFront Origin Identity access to support private content", + "Effect":"Allow", + "Principal":{"CanonicalUser":"${s3CanonicalUserId}"}, + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::${bucketName}/*" + } + ] + } + ` + + return s3 + .putBucketPolicy({ + Bucket: bucketName, + Policy: policy.replace(/(\r\n|\n|\r|\t)/gm, '') + }) + .promise() +} diff --git a/lib/index.js b/lib/index.js index 430cbd2..bd64ccd 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,13 +1,25 @@ const parseInputOrigins = require('./parseInputOrigins') const getDefaultCacheBehavior = require('./getDefaultCacheBehavior') const createOriginAccessIdentity = require('./createOriginAccessIdentity') +const grantCloudFrontBucketAccess = require('./grantCloudFrontBucketAccess') const servePrivateContentEnabled = (inputs) => inputs.origins.some((origin) => { return origin && origin.private === true }) -const createCloudFrontDistribution = async (cf, inputs) => { +const updateBucketsPolicies = async (s3, origins, s3CanonicalUserId) => { + // update bucket policies with cloudfront access + const bucketNames = origins.Items.filter((origin) => origin.S3OriginConfig).map( + (origin) => origin.Id + ) + + return Promise.all( + bucketNames.map((bucketName) => grantCloudFrontBucketAccess(s3, bucketName, s3CanonicalUserId)) + ) +} + +const createCloudFrontDistribution = async (cf, s3, inputs) => { const params = { DistributionConfig: { CallerReference: String(Date.now()), @@ -29,13 +41,18 @@ const createCloudFrontDistribution = async (cf, inputs) => { const distributionConfig = params.DistributionConfig let originAccessIdentityId + let s3CanonicalUserId if (servePrivateContentEnabled(inputs)) { - originAccessIdentityId = await createOriginAccessIdentity(cf) + ;({ originAccessIdentityId, s3CanonicalUserId } = await createOriginAccessIdentity(cf)) } const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, { originAccessIdentityId }) + if (s3CanonicalUserId) { + await updateBucketsPolicies(s3, Origins, s3CanonicalUserId) + } + distributionConfig.Origins = Origins // set first origin declared as the default cache behavior @@ -54,7 +71,7 @@ const createCloudFrontDistribution = async (cf, inputs) => { } } -const updateCloudFrontDistribution = async (cf, distributionId, inputs) => { +const updateCloudFrontDistribution = async (cf, s3, distributionId, inputs) => { // Update logic is a bit weird... // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CloudFront.html#updateDistribution-property @@ -75,17 +92,22 @@ const updateCloudFrontDistribution = async (cf, distributionId, inputs) => { params.DistributionConfig.Enabled = inputs.enabled === false ? false : true + let s3CanonicalUserId let originAccessIdentityId if (servePrivateContentEnabled(inputs)) { // presumably it's ok to call create origin access identity again // aws api returns cached copy of what was previously created // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CloudFront.html#createCloudFrontOriginAccessIdentity-property - originAccessIdentityId = await createOriginAccessIdentity(cf) + ;({ originAccessIdentityId, s3CanonicalUserId } = await createOriginAccessIdentity(cf)) } const { Origins, CacheBehaviors } = parseInputOrigins(inputs.origins, { originAccessIdentityId }) + if (s3CanonicalUserId) { + await updateBucketsPolicies(s3, Origins, s3CanonicalUserId) + } + params.DistributionConfig.DefaultCacheBehavior = getDefaultCacheBehavior(Origins.Items[0].Id) params.DistributionConfig.Origins = Origins diff --git a/serverless.js b/serverless.js index 07859cc..1d47918 100644 --- a/serverless.js +++ b/serverless.js @@ -30,14 +30,19 @@ class CloudFront extends Component { region: inputs.region }) + const s3 = new aws.S3({ + credentials: this.context.credentials.aws, + region: inputs.region + }) + if (this.state.id) { if (!equals(this.state.origins, inputs.origins)) { this.context.debug(`Updating CloudFront distribution of ID ${this.state.id}.`) - this.state = await updateCloudFrontDistribution(cf, this.state.id, inputs) + this.state = await updateCloudFrontDistribution(cf, s3, this.state.id, inputs) } } else { this.context.debug(`Creating CloudFront distribution in the ${inputs.region} region.`) - this.state = await createCloudFrontDistribution(cf, inputs) + this.state = await createCloudFrontDistribution(cf, s3, inputs) } this.state.region = inputs.region From 6528eb95d1d3cfcc3ac5a3222c46e13570508450 Mon Sep 17 00:00:00 2001 From: danielconde Date: Thu, 29 Aug 2019 22:49:25 +0100 Subject: [PATCH 4/6] update docs --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 981bbc6..31c358a 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,25 @@ distribution: viewer-request: arn:aws:lambda:us-east-1:123:function:myFunc:version # lambda ARN including version ``` +#### Private S3 Content + +To restrict access to content that you serve from S3 you can mark as `private` your S3 origins: + +```yml +# serverless.yml + +distribution: + component: '@serverless/aws-cloudfront' + inputs: + origins: + - url: https://my-private-bucket.s3.amazonaws.com + private: true +``` + +A bucket policy will be added that grants CloudFront with access to the bucket objects. Note that it doesn't remove any existing permissions on the bucket. If users currently have permission to access the files in your bucket using Amazon S3 URLs you will need to manually remove those. + +This is documented in more detail here: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-restricting-access-to-s3.html + ### 4. Deploy ```console From e864c907bf3cdfaeda75180407746e3ea99047a6 Mon Sep 17 00:00:00 2001 From: danielconde Date: Thu, 29 Aug 2019 23:22:35 +0100 Subject: [PATCH 5/6] fix origin path --- lib/getOriginConfig.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/getOriginConfig.js b/lib/getOriginConfig.js index 16b15f4..fdecd6b 100644 --- a/lib/getOriginConfig.js +++ b/lib/getOriginConfig.js @@ -3,7 +3,7 @@ const url = require('url') module.exports = (origin, { originAccessIdentityId = '' }) => { const originUrl = typeof origin === 'string' ? origin : origin.url - const { hostname } = url.parse(originUrl) + const { hostname, pathname } = url.parse(originUrl) const originConfig = { Id: hostname, @@ -12,7 +12,7 @@ module.exports = (origin, { originAccessIdentityId = '' }) => { Quantity: 0, Items: [] }, - OriginPath: '' + OriginPath: pathname === '/' ? '' : pathname } if (originUrl.includes('s3')) { From ecb49fb75a92a5c6041b7c247b4c8289f04bf003 Mon Sep 17 00:00:00 2001 From: danielconde Date: Fri, 30 Aug 2019 21:15:13 +0100 Subject: [PATCH 6/6] fix origins with more than one cache behavior --- .../__snapshots__/custom-url-origin.test.js.snap | 8 ++++++++ __tests__/__snapshots__/s3-origin.test.js.snap | 16 ++++++++++++++++ lib/parseInputOrigins.js | 11 +++++------ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/__tests__/__snapshots__/custom-url-origin.test.js.snap b/__tests__/__snapshots__/custom-url-origin.test.js.snap index 1de4d51..4aebf38 100644 --- a/__tests__/__snapshots__/custom-url-origin.test.js.snap +++ b/__tests__/__snapshots__/custom-url-origin.test.js.snap @@ -7,6 +7,10 @@ Object { "Items": Array [], "Quantity": 0, }, + "CacheBehaviors": Object { + "Items": Array [], + "Quantity": 0, + }, "CallerReference": "1566599541192", "Comment": "", "DefaultCacheBehavior": Object { @@ -93,6 +97,10 @@ Object { exports[`Input origin as a custom url updates distribution 1`] = ` Object { "DistributionConfig": Object { + "CacheBehaviors": Object { + "Items": Array [], + "Quantity": 0, + }, "DefaultCacheBehavior": Object { "AllowedMethods": Object { "CachedMethods": Object { diff --git a/__tests__/__snapshots__/s3-origin.test.js.snap b/__tests__/__snapshots__/s3-origin.test.js.snap index 1e8e9fe..ae7b4c8 100644 --- a/__tests__/__snapshots__/s3-origin.test.js.snap +++ b/__tests__/__snapshots__/s3-origin.test.js.snap @@ -7,6 +7,10 @@ Object { "Items": Array [], "Quantity": 0, }, + "CacheBehaviors": Object { + "Items": Array [], + "Quantity": 0, + }, "CallerReference": "1566599541192", "Comment": "", "DefaultCacheBehavior": Object { @@ -87,6 +91,10 @@ Object { "Items": Array [], "Quantity": 0, }, + "CacheBehaviors": Object { + "Items": Array [], + "Quantity": 0, + }, "CallerReference": "1566599541192", "Comment": "", "DefaultCacheBehavior": Object { @@ -167,6 +175,10 @@ Object { "Items": Array [], "Quantity": 0, }, + "CacheBehaviors": Object { + "Items": Array [], + "Quantity": 0, + }, "CallerReference": "1566599541192", "Comment": "", "DefaultCacheBehavior": Object { @@ -243,6 +255,10 @@ Object { exports[`S3 origins When origin is an S3 bucket URL updates distribution 1`] = ` Object { "DistributionConfig": Object { + "CacheBehaviors": Object { + "Items": Array [], + "Quantity": 0, + }, "DefaultCacheBehavior": Object { "AllowedMethods": Object { "CachedMethods": Object { diff --git a/lib/parseInputOrigins.js b/lib/parseInputOrigins.js index 0b75d6b..4052b66 100644 --- a/lib/parseInputOrigins.js +++ b/lib/parseInputOrigins.js @@ -12,7 +12,11 @@ module.exports = (origins, options) => { Quantity: 0, Items: [] } - let distributionCacheBehaviors + + const distributionCacheBehaviors = { + Quantity: 0, + Items: [] + } for (const origin of origins) { const originConfig = getOriginConfig(origin, options) @@ -44,11 +48,6 @@ module.exports = (origins, options) => { }) }) - distributionCacheBehaviors = { - Quantity: 0, - Items: [] - } - distributionCacheBehaviors.Quantity = distributionCacheBehaviors.Quantity + 1 distributionCacheBehaviors.Items.push(cacheBehavior) }