From 015065d02c1d06659dc1d1a7258b89f9415c3721 Mon Sep 17 00:00:00 2001 From: Daniel Phang Date: Tue, 17 Nov 2020 12:23:57 -0800 Subject: [PATCH] =?UTF-8?q?feat(lambda-at-edge,=20nextjs-component,=20s3-s?= =?UTF-8?q?tatic-assets):=20support=20ver=E2=80=A6=20(#803)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cypress/integration/pages.test.ts | 24 +++ packages/libs/lambda-at-edge/src/build.ts | 21 ++- .../lambda-at-edge/src/default-handler.ts | 28 ++-- .../lambda-at-edge/tests/build/build.test.ts | 9 +- .../build/simple-app-fixture/.next/BUILD_ID | 2 +- .../default-handler-origin-response.test.ts | 9 +- .../default-handler-with-basepath.test.ts | 8 +- .../default-handler-with-basic-auth.test.ts | 2 +- .../default-handler/default-handler.test.ts | 8 +- .../tests/mocks/s3/aws-sdk-s3-client.mock.ts | 9 +- packages/libs/s3-static-assets/src/index.ts | 94 ++++++++++- .../s3-static-assets/src/lib/constants.ts | 3 + packages/libs/s3-static-assets/src/lib/s3.ts | 110 +++++++++++- .../s3-static-assets/tests/aws-sdk.mock.ts | 18 +- .../tests/delete-old-static-assets.test.ts | 157 ++++++++++++++++++ .../assets/basepath/BUILD_ID | 1 + .../todos/terms.html | 0 .../todos/terms/[section].html | 0 .../todos/terms/a.html | 0 .../todos/terms/a.json | 0 .../todos/terms/b.html | 0 .../todos/terms/b.json | 0 .../.serverless_nextjs/assets/BUILD_ID | 1 + .../{ => zsWqBqLjpgRmswfQomanp}/index.html | 0 .../todos/terms.html | 0 .../todos/terms/[section].html | 0 .../todos/terms/a.html | 0 .../todos/terms/a.json | 0 .../todos/terms/b.html | 0 .../todos/terms/b.json | 0 .../tests/upload-assets-from-build.test.ts | 15 +- .../__tests__/custom-inputs.test.ts | 20 ++- .../nextjs-component/__tests__/deploy.test.ts | 29 +++- .../fixtures/simple-app/.next/BUILD_ID | 2 +- .../split-app/nextConfigDir/.next/BUILD_ID | 2 +- .../nextjs-component/src/component.ts | 9 + 36 files changed, 517 insertions(+), 64 deletions(-) create mode 100644 packages/libs/s3-static-assets/tests/delete-old-static-assets.test.ts create mode 100644 packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/BUILD_ID rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/[section].html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/a.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/a.json (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/b.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/b.json (100%) create mode 100644 packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/BUILD_ID rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/index.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/[section].html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/a.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/a.json (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/b.html (100%) rename packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/{ => zsWqBqLjpgRmswfQomanp}/todos/terms/b.json (100%) diff --git a/packages/e2e-tests/next-app-dynamic-routes/cypress/integration/pages.test.ts b/packages/e2e-tests/next-app-dynamic-routes/cypress/integration/pages.test.ts index fbba6abc36..d803338134 100644 --- a/packages/e2e-tests/next-app-dynamic-routes/cypress/integration/pages.test.ts +++ b/packages/e2e-tests/next-app-dynamic-routes/cypress/integration/pages.test.ts @@ -419,6 +419,30 @@ describe("Pages Tests", () => { } }); }); + + [ + { + path: "/optional-catch-all-ssg-with-fallback/not-prerendered", + param: "not-prerendered" + } + ].forEach(({ path, param }) => { + it(`serves but does not cache fallback page, then caches page ${path}`, () => { + // Ensure we hit a new page + const now = Date.now(); + const newPath = `${path}-${now}`; + + // Verify first request is to fallback page + cy.request(newPath).then((response) => { + expect(response.headers["cache-control"]).to.equal( + "public, max-age=0, s-maxage=0, must-revalidate" + ); + expect(response.body).to.contain('

'); + }); + + // TODO: not sure why but I couldn't verify that subsequent request is the prerendered page, Cypress seemed to cache the fallback page response. + // However, verified manually that it works correctly. + }); + }); }); describe("Dynamic SSR page", () => { diff --git a/packages/libs/lambda-at-edge/src/build.ts b/packages/libs/lambda-at-edge/src/build.ts index 65154ea114..2c5ebd7e97 100644 --- a/packages/libs/lambda-at-edge/src/build.ts +++ b/packages/libs/lambda-at-edge/src/build.ts @@ -565,7 +565,11 @@ class Builder { * Build static assets such as client-side JS, public files, static pages, etc. * Note that the upload to S3 is done in a separate deploy step. */ - async buildStaticAssets(routesManifest: RoutesManifest) { + async buildStaticAssets( + defaultBuildManifest: OriginRequestDefaultHandlerManifest, + routesManifest: RoutesManifest + ) { + const buildId = defaultBuildManifest.buildId; const basePath = routesManifest.basePath; const nextConfigDir = this.nextConfigDir; const nextStaticDir = this.nextStaticDir; @@ -587,6 +591,12 @@ class Builder { } }; + // Copy BUILD_ID file + const copyBuildId = copyIfExists( + path.join(dotNextDirectory, "BUILD_ID"), + path.join(assetOutputDirectory, withBasePath("BUILD_ID")) + ); + const buildStaticFiles = await readDirectoryFiles( path.join(dotNextDirectory, "static") ); @@ -621,7 +631,7 @@ class Builder { const destination = path.join( assetOutputDirectory, withBasePath( - `static-pages/${(relativePageFilePath as string).replace( + `static-pages/${buildId}/${(relativePageFilePath as string).replace( /^pages\//, "" )}` @@ -665,7 +675,7 @@ class Builder { ); const destination = path.join( assetOutputDirectory, - withBasePath(path.join("static-pages", relativePageFilePath)) + withBasePath(path.join("static-pages", buildId, relativePageFilePath)) ); return copyIfExists(source, destination); @@ -687,7 +697,7 @@ class Builder { const destination = path.join( assetOutputDirectory, - withBasePath(path.join("static-pages", fallback)) + withBasePath(path.join("static-pages", buildId, fallback)) ); return copyIfExists(source, destination); @@ -727,6 +737,7 @@ class Builder { const staticDirAssets = await buildPublicOrStaticDirectory("static"); return Promise.all([ + copyBuildId, // BUILD_ID ...staticFileAssets, // .next/static ...htmlPageAssets, // prerendered html pages ...prerenderManifestJSONPropFileAssets, // SSG json files @@ -807,7 +818,7 @@ class Builder { this.dotNextDir, "routes-manifest.json" )); - await this.buildStaticAssets(routesManifest); + await this.buildStaticAssets(defaultBuildManifest, routesManifest); } /** diff --git a/packages/libs/lambda-at-edge/src/default-handler.ts b/packages/libs/lambda-at-edge/src/default-handler.ts index 99136533ce..b7153585c6 100644 --- a/packages/libs/lambda-at-edge/src/default-handler.ts +++ b/packages/libs/lambda-at-edge/src/default-handler.ts @@ -346,7 +346,7 @@ const handleOriginRequest = async ({ request.uri = request.uri.replace(basePath, ""); } } else if (isHTMLPage || hasFallback) { - s3Origin.path = `${basePath}/static-pages`; + s3Origin.path = `${basePath}/static-pages/${manifest.buildId}`; const pageName = uri === "/" ? "/index" : uri; request.uri = `${pageName}.html`; } else if (isDataReq) { @@ -356,7 +356,7 @@ const handleOriginRequest = async ({ if (pagePath === "pages/404.html") { // Request static 404 page from s3 - s3Origin.path = `${basePath}/static-pages`; + s3Origin.path = `${basePath}/static-pages/${manifest.buildId}`; request.uri = pagePath.replace("pages", ""); } else if ( pagePath === "pages/_error.js" || @@ -384,7 +384,7 @@ const handleOriginRequest = async ({ const pagePath = router(manifest)(uri); if (pagePath.endsWith(".html") && !isPreviewRequest) { - s3Origin.path = `${basePath}/static-pages`; + s3Origin.path = `${basePath}/static-pages/${manifest.buildId}`; request.uri = pagePath.replace("pages", ""); addS3HostHeader(request, normalisedS3DomainName); @@ -511,9 +511,9 @@ const handleOriginResponse = async ({ }; const s3HtmlParams = { Bucket: bucketName, - Key: `${basePath}${ - basePath === "" ? "" : "/" - }static-pages/${request.uri + Key: `${basePath}${basePath === "" ? "" : "/"}static-pages/${ + manifest.buildId + }/${request.uri .replace(`/_next/data/${manifest.buildId}/`, "") .replace(".json", ".html")}`, Body: html, @@ -537,9 +537,9 @@ const handleOriginResponse = async ({ if (!hasFallback) return response; // If route has fallback, return that page from S3, otherwise return 404 page - let s3Key = `${basePath}${basePath === "" ? "" : "/"}static-pages${ - hasFallback.fallback || "/404.html" - }`; + let s3Key = `${basePath}${basePath === "" ? "" : "/"}static-pages/${ + manifest.buildId + }${hasFallback.fallback || "/404.html"}`; const { GetObjectCommand } = await import( "@aws-sdk/client-s3/commands/GetObjectCommand" @@ -553,7 +553,9 @@ const handleOriginResponse = async ({ Key: s3Key }; - const { Body } = await s3.send(new GetObjectCommand(s3Params)); + const { Body, CacheControl } = await s3.send( + new GetObjectCommand(s3Params) + ); bodyString = await getStream.default(Body as Readable); return { @@ -570,7 +572,11 @@ const handleOriginResponse = async ({ "cache-control": [ { key: "Cache-Control", - value: "public, max-age=0, s-maxage=2678400, must-revalidate" + value: + CacheControl ?? + (hasFallback.fallback // Use cache-control from S3 response if possible, otherwise use defaults + ? "public, max-age=0, s-maxage=0, must-revalidate" // fallback should never be cached + : "public, max-age=0, s-maxage=2678400, must-revalidate") } ] }, diff --git a/packages/libs/lambda-at-edge/tests/build/build.test.ts b/packages/libs/lambda-at-edge/tests/build/build.test.ts index 227755a1bc..c872c4397c 100644 --- a/packages/libs/lambda-at-edge/tests/build/build.test.ts +++ b/packages/libs/lambda-at-edge/tests/build/build.test.ts @@ -306,7 +306,7 @@ describe("Builder Tests", () => { expect(nextStaticFiles).toEqual(["chunks"]); const staticPagesFiles = await fse.readdir( - join(outputDir, `${ASSETS_DIR}/static-pages`) + join(outputDir, `${ASSETS_DIR}/static-pages/test-build-id`) ); expect(staticPagesFiles).toEqual([ "about.html", @@ -314,6 +314,13 @@ describe("Builder Tests", () => { "index.html", "terms.html" ]); + + // Check BUILD_ID file + expect( + ( + await fse.readFile(join(outputDir, `${ASSETS_DIR}/BUILD_ID`)) + ).toString() + ).toEqual("test-build-id"); }); }); }); diff --git a/packages/libs/lambda-at-edge/tests/build/simple-app-fixture/.next/BUILD_ID b/packages/libs/lambda-at-edge/tests/build/simple-app-fixture/.next/BUILD_ID index d7dd5f1246..5ad897a7a0 100644 --- a/packages/libs/lambda-at-edge/tests/build/simple-app-fixture/.next/BUILD_ID +++ b/packages/libs/lambda-at-edge/tests/build/simple-app-fixture/.next/BUILD_ID @@ -1 +1 @@ -test-build-id +test-build-id \ No newline at end of file diff --git a/packages/libs/lambda-at-edge/tests/default-handler/default-handler-origin-response.test.ts b/packages/libs/lambda-at-edge/tests/default-handler/default-handler-origin-response.test.ts index 60c33a462f..5b72e04a4b 100644 --- a/packages/libs/lambda-at-edge/tests/default-handler/default-handler-origin-response.test.ts +++ b/packages/libs/lambda-at-edge/tests/default-handler/default-handler-origin-response.test.ts @@ -75,7 +75,8 @@ describe("Lambda@Edge origin response", () => { expect(s3Client.send).toHaveBeenCalledWith({ Command: "GetObjectCommand", Bucket: "my-bucket.s3.amazonaws.com", - Key: "static-pages/tests/prerender-manifest-fallback/[fallback].html" + Key: + "static-pages/build-id/tests/prerender-manifest-fallback/[fallback].html" }); expect(response).toEqual({ @@ -85,7 +86,7 @@ describe("Lambda@Edge origin response", () => { "cache-control": [ { key: "Cache-Control", - value: "public, max-age=0, s-maxage=2678400, must-revalidate" + value: "public, max-age=0, s-maxage=0, must-revalidate" // Fallback page shouldn't be cached as it will override the path for a just generated SSG page. } ], "content-type": [ @@ -115,7 +116,7 @@ describe("Lambda@Edge origin response", () => { expect(s3Client.send).toHaveBeenCalledWith({ Command: "GetObjectCommand", Bucket: "my-bucket.s3.amazonaws.com", - Key: "static-pages/404.html" + Key: "static-pages/build-id/404.html" }); expect(response).toEqual({ @@ -180,7 +181,7 @@ describe("Lambda@Edge origin response", () => { expect(s3Client.send).toHaveBeenNthCalledWith(2, { Command: "PutObjectCommand", Bucket: "my-bucket.s3.amazonaws.com", - Key: "static-pages/fallback/not-yet-built.html", + Key: "static-pages/build-id/fallback/not-yet-built.html", Body: "
Rendered Page
", ContentType: "text/html", CacheControl: "public, max-age=0, s-maxage=2678400, must-revalidate" diff --git a/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basepath.test.ts b/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basepath.test.ts index 0e6f9856f9..0bd0cc79bb 100644 --- a/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basepath.test.ts +++ b/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basepath.test.ts @@ -142,7 +142,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/basepath/static-pages", + path: "/basepath/static-pages/build-id", region: "us-east-1" } }); @@ -446,7 +446,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.eu-west-1.amazonaws.com", - path: "/basepath/static-pages", + path: "/basepath/static-pages/build-id", region: "eu-west-1" } }); @@ -541,7 +541,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/basepath/static-pages", + path: "/basepath/static-pages/build-id", region: "us-east-1" } }); @@ -712,7 +712,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/basepath/static-pages", + path: "/basepath/static-pages/build-id", region: "us-east-1" } }); diff --git a/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basic-auth.test.ts b/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basic-auth.test.ts index 87143d0714..ab01f04520 100644 --- a/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basic-auth.test.ts +++ b/packages/libs/lambda-at-edge/tests/default-handler/default-handler-with-basic-auth.test.ts @@ -64,7 +64,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/static-pages", + path: "/static-pages/build-id2", region: "us-east-1" } }, diff --git a/packages/libs/lambda-at-edge/tests/default-handler/default-handler.test.ts b/packages/libs/lambda-at-edge/tests/default-handler/default-handler.test.ts index b73d1a8f35..60590cdb84 100644 --- a/packages/libs/lambda-at-edge/tests/default-handler/default-handler.test.ts +++ b/packages/libs/lambda-at-edge/tests/default-handler/default-handler.test.ts @@ -148,7 +148,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/static-pages", + path: "/static-pages/build-id", region: "us-east-1" } }); @@ -247,7 +247,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/static-pages", + path: "/static-pages/build-id", region: "us-east-1" } }); @@ -541,7 +541,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.eu-west-1.amazonaws.com", - path: "/static-pages", + path: "/static-pages/build-id", region: "eu-west-1" } }); @@ -766,7 +766,7 @@ describe("Lambda@Edge", () => { s3: { authMethod: "origin-access-identity", domainName: "my-bucket.s3.amazonaws.com", - path: "/static-pages", + path: "/static-pages/build-id", region: "us-east-1" } }); diff --git a/packages/libs/lambda-at-edge/tests/mocks/s3/aws-sdk-s3-client.mock.ts b/packages/libs/lambda-at-edge/tests/mocks/s3/aws-sdk-s3-client.mock.ts index a8dd9a2392..5f7e75f98a 100644 --- a/packages/libs/lambda-at-edge/tests/mocks/s3/aws-sdk-s3-client.mock.ts +++ b/packages/libs/lambda-at-edge/tests/mocks/s3/aws-sdk-s3-client.mock.ts @@ -2,8 +2,15 @@ import { Readable } from "stream"; export const mockSend = jest.fn((input) => { if (input.Command === "GetObjectCommand") { + // Simulate fallback page cache control headers + const isFallback = /\[.*]/.test(input.Key as string); + const cacheControl = isFallback + ? "public, max-age=0, s-maxage=0, must-revalidate" + : "public, max-age=0, s-maxage=2678400, must-revalidate"; + return { - Body: Readable.from(["S3Body"]) + Body: Readable.from(["S3Body"]), + CacheControl: cacheControl }; } else { return {}; diff --git a/packages/libs/s3-static-assets/src/index.ts b/packages/libs/s3-static-assets/src/index.ts index 9dd3e20f11..e2bcb84ba1 100644 --- a/packages/libs/s3-static-assets/src/index.ts +++ b/packages/libs/s3-static-assets/src/index.ts @@ -5,6 +5,7 @@ import readDirectoryFiles from "./lib/readDirectoryFiles"; import filterOutDirectories from "./lib/filterOutDirectories"; import { IMMUTABLE_CACHE_CONTROL_HEADER, + SERVER_NO_CACHE_CACHE_CONTROL_HEADER, SERVER_CACHE_CONTROL_HEADER } from "./lib/constants"; import S3ClientFactory, { Credentials } from "./lib/s3"; @@ -51,6 +52,17 @@ const uploadStaticAssetsFromBuild = async ( "assets" ); + // Upload BUILD_ID file to represent the current build ID to be uploaded. This is for metadata and also used for deleting old versioned files. + const buildIdPath = path.join( + assetsOutputDirectory, + normalizedBasePath, + "BUILD_ID" + ); + const buildIdUpload = s3.uploadFile({ + s3Key: pathToPosix(path.join(normalizedBasePath, "BUILD_ID")), + filePath: buildIdPath + }); + // Upload Next.js static files const nextStaticFiles = await readDirectoryFiles( @@ -104,11 +116,21 @@ const uploadStaticAssetsFromBuild = async ( path.relative(assetsOutputDirectory, fileItem.path) ); - return s3.uploadFile({ - s3Key, - filePath: fileItem.path, - cacheControl: SERVER_CACHE_CONTROL_HEADER - }); + // Dynamic fallback HTML pages should never be cached as it will override actual pages once generated and stored in S3. + const isDynamicFallback = /\[.*]/.test(s3Key); + if (isDynamicFallback) { + return s3.uploadFile({ + s3Key, + filePath: fileItem.path, + cacheControl: SERVER_NO_CACHE_CACHE_CONTROL_HEADER + }); + } else { + return s3.uploadFile({ + s3Key, + filePath: fileItem.path, + cacheControl: SERVER_CACHE_CONTROL_HEADER + }); + } }); // Upload user static and public files @@ -142,7 +164,8 @@ const uploadStaticAssetsFromBuild = async ( ...nextStaticFilesUploads, ...nextDataFilesUploads, ...htmlPagesUploads, - ...publicAndStaticUploads + ...publicAndStaticUploads, + buildIdUpload ]); }; @@ -334,4 +357,61 @@ const uploadStaticAssets = async ( return Promise.all(allUploads); }; -export { uploadStaticAssetsFromBuild, uploadStaticAssets }; +type DeleteOldStaticAssetsOptions = { + bucketName: string; + basePath: string; + credentials: Credentials; +}; + +/** + * Remove old static assets from S3 bucket. This reads the existing BUILD_ID file from S3. + * If it exists, it removes all versioned assets except that BUILD_ID, in order to save S3 storage costs. + * @param options + */ +const deleteOldStaticAssets = async ( + options: DeleteOldStaticAssetsOptions +): Promise => { + const { bucketName, basePath } = options; + + const normalizedBasePathPrefix = basePath ? basePath.slice(1) + "/" : ""; + + const s3 = await S3ClientFactory({ + bucketName, + credentials: options.credentials + }); + + // Get BUILD_ID file from S3 if it exists + let buildId = await s3.getFile({ + key: normalizedBasePathPrefix + "BUILD_ID" + }); + + // If above exists, remove all versioned assets that are not BUILD_ID file (we don't remove unversioned pages static-pages as those are the previous way) + if (buildId) { + // Delete old _next/data versions except for buildId + const deleteNextDataFiles = s3.deleteFilesByPattern({ + prefix: `${normalizedBasePathPrefix}_next/data`, + pattern: new RegExp(`${normalizedBasePathPrefix}_next/data/.+/`), // Ensure to only delete versioned directories + excludePattern: new RegExp( + `${normalizedBasePathPrefix}_next/data/${buildId}/` + ) // Don't delete given build ID + }); + + // Delete old static-pages versions except for buildId + const deleteStaticPageFiles = s3.deleteFilesByPattern({ + prefix: `${normalizedBasePathPrefix}static-pages`, + pattern: new RegExp(`${normalizedBasePathPrefix}static-pages/.+/`), // Ensure to only delete versioned directories + excludePattern: new RegExp( + `${normalizedBasePathPrefix}static-pages/${buildId}/` + ) // Don't delete given build ID + }); + + // Run deletion tasks in parallel (safe since they have different prefixes) + await Promise.all([deleteNextDataFiles, deleteStaticPageFiles]); + } +}; + +export { + deleteOldStaticAssets, + uploadStaticAssetsFromBuild, + uploadStaticAssets +}; diff --git a/packages/libs/s3-static-assets/src/lib/constants.ts b/packages/libs/s3-static-assets/src/lib/constants.ts index 16e6869325..63bcd61434 100644 --- a/packages/libs/s3-static-assets/src/lib/constants.ts +++ b/packages/libs/s3-static-assets/src/lib/constants.ts @@ -7,4 +7,7 @@ export const SERVER_CACHE_CONTROL_HEADER = export const DEFAULT_PUBLIC_DIR_CACHE_CONTROL = "public, max-age=31536000, must-revalidate"; +export const SERVER_NO_CACHE_CACHE_CONTROL_HEADER = + "public, max-age=0, s-maxage=0, must-revalidate"; + export const DEFAULT_PUBLIC_DIR_CACHE_REGEX = /\.(gif|jpe?g|jp2|tiff|png|webp|bmp|svg|ico)$/i; diff --git a/packages/libs/s3-static-assets/src/lib/s3.ts b/packages/libs/s3-static-assets/src/lib/s3.ts index e541f4fe41..f6feac7e58 100644 --- a/packages/libs/s3-static-assets/src/lib/s3.ts +++ b/packages/libs/s3-static-assets/src/lib/s3.ts @@ -1,6 +1,8 @@ import getMimeType from "./getMimeType"; import fse from "fs-extra"; -import AWS from "aws-sdk"; +import AWS, { AWSError, S3 } from "aws-sdk"; +import { PromiseResult } from "aws-sdk/lib/request"; +import { ObjectList } from "aws-sdk/clients/s3"; type S3ClientFactoryOptions = { bucketName: string; @@ -13,10 +15,30 @@ type UploadFileOptions = { s3Key?: string; }; +type DeleteFilesByPatternOptions = { + prefix: string; + pattern: RegExp; + excludePattern?: RegExp; +}; + +type GetFileOptions = { + key: string; +}; + export type S3Client = { uploadFile: ( options: UploadFileOptions ) => Promise; + /** + * Delete all files in S3 given the pattern. + * @param options + */ + deleteFilesByPattern: (options: DeleteFilesByPatternOptions) => Promise; + /** + * Get file in S3 given the key and read it into a string. + * @param options + */ + getFile: (options: GetFileOptions) => Promise; }; export type Credentials = { @@ -64,6 +86,92 @@ export default async ({ CacheControl: cacheControl || undefined }) .promise(); + }, + deleteFilesByPattern: async ( + options: DeleteFilesByPatternOptions + ): Promise => { + const { prefix, pattern, excludePattern } = options; + + // 1. Get all objects by given prefix and matching the pattern, but excluding a pattern. + const foundKeys: string[] = []; + let continuationToken = undefined; // needed to paginate through all objects + + while (true) { + const data: PromiseResult = await s3 + .listObjectsV2({ + Bucket: bucketName, + Prefix: prefix, + ContinuationToken: continuationToken + }) + .promise(); + + // Push all objects + const contents: ObjectList = data.Contents ?? []; + contents.forEach(function (content) { + if (content.Key) { + const key = content.Key; + + // Match pattern and does not match exclude pattern + if ( + pattern.test(key) && + (!excludePattern || !excludePattern.test(key)) + ) { + foundKeys.push(content.Key); + } + } + }); + + // Continue listing since ListObjectsV2 gets up to 1000 objects at a time + if (data.IsTruncated) { + continuationToken = data.NextContinuationToken; + } else { + break; + } + } + + const maxKeysToDelete = 1000; // From https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + + // 2. Delete all the objects in batch mode + let start = 0; + while (start < foundKeys.length) { + const objects = []; + for ( + let i = start; + i < start + maxKeysToDelete && i < foundKeys.length; + i++ + ) { + objects.push({ + Key: foundKeys[i] + }); + } + + await s3 + .deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: objects + } + }) + .promise(); + + start += maxKeysToDelete; + } + }, + getFile: async (options: GetFileOptions): Promise => { + try { + const data = await s3 + .getObject({ + Bucket: bucketName, + Key: options.key + }) + .promise(); + + return data.Body?.toString("utf-8"); + } catch (e) { + if (e.code === "NoSuchKey") { + return undefined; + } + } } }; }; diff --git a/packages/libs/s3-static-assets/tests/aws-sdk.mock.ts b/packages/libs/s3-static-assets/tests/aws-sdk.mock.ts index 7c1c6197c6..56a58f9d78 100644 --- a/packages/libs/s3-static-assets/tests/aws-sdk.mock.ts +++ b/packages/libs/s3-static-assets/tests/aws-sdk.mock.ts @@ -2,6 +2,12 @@ declare module "aws-sdk" { const mockUpload: jest.Mock; const mockGetBucketAccelerateConfiguration: jest.Mock; const mockGetBucketAccelerateConfigurationPromise: jest.Mock; + const mockGetObject: jest.Mock; + const mockGetObjectPromise: jest.Mock; + const mockListObjectsV2: jest.Mock; + const mockListObjectsV2Promise: jest.Mock; + const mockDeleteObjects: jest.Mock; + const mockDeleteObjectsPromise: jest.Mock; } const promisifyMock = (mockFn: jest.Mock): jest.Mock => { @@ -19,9 +25,19 @@ export const mockGetBucketAccelerateConfigurationPromise = promisifyMock( Status: "Suspended" })); +export const mockGetObject = jest.fn(); +export const mockGetObjectPromise = promisifyMock(mockGetObject); +export const mockListObjectsV2 = jest.fn(); +export const mockListObjectsV2Promise = promisifyMock(mockListObjectsV2); +export const mockDeleteObjects = jest.fn(); +export const mockDeleteObjectsPromise = promisifyMock(mockDeleteObjects); + const MockS3 = jest.fn(() => ({ upload: mockUpload, - getBucketAccelerateConfiguration: mockGetBucketAccelerateConfiguration + getBucketAccelerateConfiguration: mockGetBucketAccelerateConfiguration, + getObject: mockGetObject, + listObjectsV2: mockListObjectsV2, + deleteObjects: mockDeleteObjects })); export default { diff --git a/packages/libs/s3-static-assets/tests/delete-old-static-assets.test.ts b/packages/libs/s3-static-assets/tests/delete-old-static-assets.test.ts new file mode 100644 index 0000000000..99ad6843b3 --- /dev/null +++ b/packages/libs/s3-static-assets/tests/delete-old-static-assets.test.ts @@ -0,0 +1,157 @@ +import { deleteOldStaticAssets } from "../src/index"; +import AWS, { + mockGetObject, + mockGetObjectPromise, + mockListObjectsV2, + mockDeleteObjects +} from "aws-sdk"; + +// unfortunately can't use __mocks__ because aws-sdk is being mocked in other +// packages in the monorepo +// https://github.com/facebook/jest/issues/2070 +jest.mock("aws-sdk", () => require("./aws-sdk.mock")); + +const deleteOldAssets = (basePath?: string): Promise => { + return deleteOldStaticAssets({ + bucketName: "test-bucket-name", + basePath: basePath || "", + credentials: { + accessKeyId: "fake-access-key", + secretAccessKey: "fake-secret-key", + sessionToken: "fake-session-token" + } + }); +}; + +describe.each` + basePath + ${""} + ${"/basepath"} +`( + "Delete old static assets tests for basePath: [$basePath]", + ({ basePath }) => { + let consoleWarnSpy: jest.SpyInstance; + const basePathPrefix = basePath === "" ? "" : basePath.slice(1) + "/"; + + beforeEach(() => { + consoleWarnSpy = jest.spyOn(console, "warn").mockReturnValue(); + }); + + afterEach(() => { + consoleWarnSpy.mockRestore(); + }); + + it("does not delete static assets when no BUILD_ID exists", async () => { + mockGetObjectPromise.mockRejectedValue({ + code: "NoSuchKey" + }); + + await deleteOldAssets(basePath); + + expect(mockGetObject).toBeCalledTimes(1); + expect(mockGetObject).toBeCalledWith({ + Bucket: "test-bucket-name", + Key: basePathPrefix + "BUILD_ID" + }); + + expect(mockListObjectsV2).toBeCalledTimes(0); + expect(mockDeleteObjects).toBeCalledTimes(0); + }); + + it("does not delete static assets when BUILD_ID exists but listed objects are empty", async () => { + mockGetObjectPromise.mockResolvedValue({ + Body: "test-build-id" + }); + + mockListObjectsV2.mockImplementation(() => { + return { promise: () => Promise.resolve({ Contents: [] }) }; + }); + + await deleteOldAssets(basePath); + + expect(mockListObjectsV2).toBeCalledTimes(2); + expect(mockDeleteObjects).toBeCalledTimes(0); + }); + + it("deletes static assets when BUILD_ID exists", async () => { + mockGetObjectPromise.mockResolvedValue({ + Body: "test-build-id" + }); + + mockListObjectsV2.mockImplementation((object) => { + if (object.Prefix === basePathPrefix + "static-pages") { + return { + promise: () => + Promise.resolve({ + Contents: [ + { + Key: basePathPrefix + "static-pages/prev-build-id/page.html" + }, + { + Key: basePathPrefix + "static-pages/test-build-id/page.html" + }, + { Key: basePathPrefix + "static-pages/page.html" } // Backwards compatibility, never delete existing pages in root + ] + }) + }; + } else if (object.Prefix === basePathPrefix + "_next/data") { + return { + promise: () => + Promise.resolve({ + Contents: [ + { + Key: basePathPrefix + "_next/data/prev-build-id/page.json" + }, + { Key: basePathPrefix + "_next/data/test-build-id/page.json" } + ] + }) + }; + } + }); + + await deleteOldAssets(basePath); + + expect(AWS.S3).toBeCalledWith({ + accessKeyId: "fake-access-key", + secretAccessKey: "fake-secret-key", + sessionToken: "fake-session-token" + }); + + expect(mockDeleteObjects).toBeCalledTimes(2); + + expect(mockListObjectsV2).toBeCalledWith({ + Bucket: "test-bucket-name", + Prefix: basePathPrefix + "static-pages", + ContinuationToken: undefined + }); + + expect(mockListObjectsV2).toBeCalledWith({ + Bucket: "test-bucket-name", + Prefix: basePathPrefix + "_next/data", + ContinuationToken: undefined + }); + + expect(mockDeleteObjects).toBeCalledTimes(2); + + // Expect only prev-build-id is deleted, not test-build-id which is in BUILD_ID (current build ID in S3) + + expect(mockDeleteObjects).toBeCalledWith({ + Bucket: "test-bucket-name", + Delete: { + Objects: [ + { Key: basePathPrefix + "static-pages/prev-build-id/page.html" } + ] + } + }); + + expect(mockDeleteObjects).toBeCalledWith({ + Bucket: "test-bucket-name", + Delete: { + Objects: [ + { Key: basePathPrefix + "_next/data/prev-build-id/page.json" } + ] + } + }); + }); + } +); diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/BUILD_ID b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/BUILD_ID new file mode 100644 index 0000000000..7bba726357 --- /dev/null +++ b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/BUILD_ID @@ -0,0 +1 @@ +zsWqBqLjpgRmswfQomanp \ No newline at end of file diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/[section].html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/[section].html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/[section].html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/[section].html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/a.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/a.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/a.json b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.json similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/a.json rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.json diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/b.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/b.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/b.json b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.json similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/todos/terms/b.json rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.json diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/BUILD_ID b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/BUILD_ID new file mode 100644 index 0000000000..7bba726357 --- /dev/null +++ b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/BUILD_ID @@ -0,0 +1 @@ +zsWqBqLjpgRmswfQomanp \ No newline at end of file diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/index.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/index.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/index.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/index.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/[section].html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/[section].html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/[section].html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/[section].html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/a.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/a.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/a.json b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.json similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/a.json rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.json diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/b.html b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.html similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/b.html rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.html diff --git a/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/b.json b/packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.json similarity index 100% rename from packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/todos/terms/b.json rename to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.json diff --git a/packages/libs/s3-static-assets/tests/upload-assets-from-build.test.ts b/packages/libs/s3-static-assets/tests/upload-assets-from-build.test.ts index d5832eb345..e1cddc89a7 100644 --- a/packages/libs/s3-static-assets/tests/upload-assets-from-build.test.ts +++ b/packages/libs/s3-static-assets/tests/upload-assets-from-build.test.ts @@ -2,6 +2,7 @@ import path from "path"; import { uploadStaticAssetsFromBuild } from "../src/index"; import { IMMUTABLE_CACHE_CONTROL_HEADER, + SERVER_NO_CACHE_CACHE_CONTROL_HEADER, SERVER_CACHE_CONTROL_HEADER } from "../src/lib/constants"; import AWS, { @@ -98,10 +99,6 @@ describe("Upload tests from build", () => { ); expect(AWS.S3).toBeCalledTimes(1); }); - - describe("when no public or static directory exists", () => { - it("upload does not crash", () => upload("./fixtures/app-no-public-dir")); - }); }); describe.each` @@ -135,7 +132,7 @@ describe.each` it("uploads HTML pages in static-pages", async () => { expect(mockUpload).toBeCalledWith( expect.objectContaining({ - Key: "static-pages/todos/terms.html", + Key: "static-pages/zsWqBqLjpgRmswfQomanp/todos/terms.html", ContentType: "text/html", CacheControl: SERVER_CACHE_CONTROL_HEADER }) @@ -143,15 +140,15 @@ describe.each` expect(mockUpload).toBeCalledWith( expect.objectContaining({ - Key: "static-pages/todos/terms/[section].html", + Key: "static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/[section].html", ContentType: "text/html", - CacheControl: SERVER_CACHE_CONTROL_HEADER + CacheControl: SERVER_NO_CACHE_CACHE_CONTROL_HEADER }) ); expect(mockUpload).toBeCalledWith( expect.objectContaining({ - Key: "static-pages/todos/terms/a.html", + Key: "static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/a.html", ContentType: "text/html", CacheControl: SERVER_CACHE_CONTROL_HEADER }) @@ -159,7 +156,7 @@ describe.each` expect(mockUpload).toBeCalledWith( expect.objectContaining({ - Key: "static-pages/todos/terms/b.html", + Key: "static-pages/zsWqBqLjpgRmswfQomanp/todos/terms/b.html", ContentType: "text/html", CacheControl: SERVER_CACHE_CONTROL_HEADER }) diff --git a/packages/serverless-components/nextjs-component/__tests__/custom-inputs.test.ts b/packages/serverless-components/nextjs-component/__tests__/custom-inputs.test.ts index 96c502e028..6fde13361b 100644 --- a/packages/serverless-components/nextjs-component/__tests__/custom-inputs.test.ts +++ b/packages/serverless-components/nextjs-component/__tests__/custom-inputs.test.ts @@ -205,14 +205,13 @@ describe("Custom inputs", () => { }); it("uploads static assets to S3 correctly", () => { - expect(mockUpload).toBeCalledTimes(11); + expect(mockUpload).toBeCalledTimes(12); [ - "static-pages/index.html", - "static-pages/terms.html", - "static-pages/404.html", - "static-pages/about.html", - "static-pages/blog/[post].html" + "static-pages/test-build-id/index.html", + "static-pages/test-build-id/terms.html", + "static-pages/test-build-id/404.html", + "static-pages/test-build-id/about.html" ].forEach((file) => { expect(mockUpload).toBeCalledWith( expect.objectContaining({ @@ -223,6 +222,15 @@ describe("Custom inputs", () => { ); }); + ["static-pages/test-build-id/blog/[post].html"].forEach((file) => { + expect(mockUpload).toBeCalledWith( + expect.objectContaining({ + Key: file, + CacheControl: "public, max-age=0, s-maxage=0, must-revalidate" + }) + ); + }); + ["_next/static/test-build-id/placeholder.js"].forEach((file) => { expect(mockUpload).toBeCalledWith( expect.objectContaining({ diff --git a/packages/serverless-components/nextjs-component/__tests__/deploy.test.ts b/packages/serverless-components/nextjs-component/__tests__/deploy.test.ts index 2955a79ce9..523c6c8276 100644 --- a/packages/serverless-components/nextjs-component/__tests__/deploy.test.ts +++ b/packages/serverless-components/nextjs-component/__tests__/deploy.test.ts @@ -231,14 +231,21 @@ describe("deploy tests", () => { }); it("uploads static assets to S3 correctly", () => { - expect(mockUpload).toBeCalledTimes(12); + expect(mockUpload).toBeCalledTimes(13); + + ["BUILD_ID"].forEach((file) => { + expect(mockUpload).toBeCalledWith( + expect.objectContaining({ + Key: file + }) + ); + }); [ - "static-pages/index.html", - "static-pages/terms.html", - "static-pages/404.html", - "static-pages/about.html", - "static-pages/blog/[post].html" + "static-pages/test-build-id/index.html", + "static-pages/test-build-id/terms.html", + "static-pages/test-build-id/404.html", + "static-pages/test-build-id/about.html" ].forEach((file) => { expect(mockUpload).toBeCalledWith( expect.objectContaining({ @@ -248,6 +255,16 @@ describe("deploy tests", () => { ); }); + // Fallback page is never cached in S3 + ["static-pages/test-build-id/blog/[post].html"].forEach((file) => { + expect(mockUpload).toBeCalledWith( + expect.objectContaining({ + Key: file, + CacheControl: "public, max-age=0, s-maxage=0, must-revalidate" + }) + ); + }); + [ "_next/static/chunks/chunk1.js", "_next/static/test-build-id/placeholder.js" diff --git a/packages/serverless-components/nextjs-component/__tests__/fixtures/simple-app/.next/BUILD_ID b/packages/serverless-components/nextjs-component/__tests__/fixtures/simple-app/.next/BUILD_ID index d7dd5f1246..5ad897a7a0 100644 --- a/packages/serverless-components/nextjs-component/__tests__/fixtures/simple-app/.next/BUILD_ID +++ b/packages/serverless-components/nextjs-component/__tests__/fixtures/simple-app/.next/BUILD_ID @@ -1 +1 @@ -test-build-id +test-build-id \ No newline at end of file diff --git a/packages/serverless-components/nextjs-component/__tests__/fixtures/split-app/nextConfigDir/.next/BUILD_ID b/packages/serverless-components/nextjs-component/__tests__/fixtures/split-app/nextConfigDir/.next/BUILD_ID index d7dd5f1246..5ad897a7a0 100644 --- a/packages/serverless-components/nextjs-component/__tests__/fixtures/split-app/nextConfigDir/.next/BUILD_ID +++ b/packages/serverless-components/nextjs-component/__tests__/fixtures/split-app/nextConfigDir/.next/BUILD_ID @@ -1 +1 @@ -test-build-id +test-build-id \ No newline at end of file diff --git a/packages/serverless-components/nextjs-component/src/component.ts b/packages/serverless-components/nextjs-component/src/component.ts index 68831b5400..53008adae1 100644 --- a/packages/serverless-components/nextjs-component/src/component.ts +++ b/packages/serverless-components/nextjs-component/src/component.ts @@ -9,6 +9,7 @@ import { RoutesManifest } from "@sls-next/lambda-at-edge/types"; import { + deleteOldStaticAssets, uploadStaticAssetsFromBuild, uploadStaticAssets } from "@sls-next/s3-static-assets"; @@ -294,6 +295,14 @@ class NextjsComponent extends Component { region: bucketRegion }); + // If new BUILD_ID file is present, remove all versioned assets but the existing build ID's assets, to save S3 storage costs. + // After deployment, only the new and previous build ID's assets are present. We still need previous build assets as it takes time to propagate the Lambda. + await deleteOldStaticAssets({ + bucketName: bucketOutputs.name, + basePath: routesManifest.basePath, + credentials: this.context.credentials.aws + }); + // This input is intentionally undocumented but it acts a short-term killswitch in case of any issues with uploading from the built assets. // TODO: remove once proven stable. if (inputs.uploadStaticAssetsFromBuild ?? true) {