fix: don't deploy metadata files to CDN (#2104)

* fix: tRaCe

* chore: add more metadata files for deletion

* refactor: move concrete file removal logic into it's own function

* chore: make hidden paths consistent

* test: update snapshot and add unit test for file removal

* chore: add code comment about timing of metadata files removal

* fix: publish 404 and 500 to cdn

* test: don't delete BUILD_ID in test env

* test: update snapshot

* fix: typescript gods

* chore: dummy commit

* chore: dummy commit2

* chore: dummy commit3

* test: update snapshot

* test: use different dedicated env var to skip deleting BUILD_ID

* test: update utils.spec.ts assertions

* test: keep all the metadata for e2e tests

* chore: update comment

* perf: delete files concurrently instead of 1 by 1

* chore: update comment

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

Author: pieh

packages/runtime/src/constants.ts

@@ -1,3 +1,5 @@
+import destr from 'destr'
+
 export const HANDLER_FUNCTION_NAME = '___netlify-handler'
 export const ODB_FUNCTION_NAME = '___netlify-odb-handler'
 export const IMAGE_FUNCTION_NAME = '_ipx'
@@ -7,18 +9,29 @@ export const HANDLER_FUNCTION_TITLE = 'Next.js SSR handler'
 export const ODB_FUNCTION_TITLE = 'Next.js ISR handler'
 export const IMAGE_FUNCTION_TITLE = 'next/image handler'
 // These are paths in .next that shouldn't be publicly accessible
-export const HIDDEN_PATHS = [
-  '/cache/*',
-  '/server/*',
-  '/serverless/*',
-  '/trace',
-  '/traces',
-  '/routes-manifest.json',
-  '/build-manifest.json',
-  '/prerender-manifest.json',
-  '/react-loadable-manifest.json',
-  '/BUILD_ID',
-]
+export const HIDDEN_PATHS = destr(process.env.NEXT_KEEP_METADATA_FILES)
+  ? []
+  : [
+      '/cache',
+      '/server',
+      '/serverless',
+      '/trace',
+      '/traces',
+      '/routes-manifest.json',
+      '/build-manifest.json',
+      '/prerender-manifest.json',
+      '/react-loadable-manifest.json',
+      '/BUILD_ID',
+      '/app-build-manifest.json',
+      '/app-path-routes-manifest.json',
+      '/export-marker.json',
+      '/images-manifest.json',
+      '/next-server.js.nft.json',
+      '/package.json',
+      '/prerender-manifest.js',
+      '/required-server-files.json',
+      '/static-manifest.json',
+    ]
 
 export const ODB_FUNCTION_PATH = `/.netlify/builders/${ODB_FUNCTION_NAME}`
 export const HANDLER_FUNCTION_PATH = `/.netlify/functions/${HANDLER_FUNCTION_NAME}`

packages/runtime/src/helpers/files.ts

@@ -2,15 +2,26 @@ import { cpus } from 'os'
 
 import type { NetlifyConfig } from '@netlify/build'
 import { yellowBright } from 'chalk'
-import { existsSync, readJson, move, copy, writeJson, readFile, writeFile, ensureDir, readFileSync } from 'fs-extra'
+import {
+  existsSync,
+  readJson,
+  move,
+  copy,
+  writeJson,
+  readFile,
+  writeFile,
+  ensureDir,
+  readFileSync,
+  remove,
+} from 'fs-extra'
 import globby from 'globby'
 import { PrerenderManifest } from 'next/dist/build'
 import { outdent } from 'outdent'
 import pLimit from 'p-limit'
 import { join, resolve, dirname } from 'pathe'
 import slash from 'slash'
 
-import { MINIMUM_REVALIDATE_SECONDS, DIVIDER } from '../constants'
+import { MINIMUM_REVALIDATE_SECONDS, DIVIDER, HIDDEN_PATHS } from '../constants'
 
 import { NextConfig } from './config'
 import { loadPrerenderManifest } from './edge'
@@ -138,8 +149,8 @@ export const moveStaticPages = async ({
       console.warn('Error moving file', source, error)
     }
   }
-  // Move all static files, except error documents and nft manifests
-  const pages = await globby(['{app,pages}/**/*.{html,json,rsc}', '!**/(500|404|*.js.nft).{html,json}'], {
+  // Move all static files, except nft manifests
+  const pages = await globby(['{app,pages}/**/*.{html,json,rsc}', '!**/*.js.nft.{html,json}'], {
     cwd: outputDir,
     dot: true,
   })
@@ -467,3 +478,15 @@ export const movePublicFiles = async ({
     await copy(publicDir, `${publish}${basePath}/`)
   }
 }
+
+export const removeMetadataFiles = async (publish: string) => {
+  // Limit concurrent deletions to number of cpus or 2 if there is only 1
+  const limit = pLimit(Math.max(2, cpus().length))
+
+  const removePromises = HIDDEN_PATHS.map((HIDDEN_PATH) => {
+    const pathToDelete = join(publish, HIDDEN_PATH)
+    return limit(() => remove(pathToDelete))
+  })
+
+  await Promise.all(removePromises)
+}

packages/runtime/src/helpers/redirects.ts

@@ -7,7 +7,7 @@ import type { PrerenderManifest, SsgRoute } from 'next/dist/build'
 import { outdent } from 'outdent'
 import { join } from 'pathe'
 
-import { HANDLER_FUNCTION_PATH, HIDDEN_PATHS, ODB_FUNCTION_PATH } from '../constants'
+import { HANDLER_FUNCTION_PATH, ODB_FUNCTION_PATH } from '../constants'
 
 import { isAppDirRoute, loadAppPathRoutesManifest } from './edge'
 import { getMiddleware } from './files'
@@ -27,14 +27,6 @@ import {
 const matchesMiddleware = (middleware: Array<string>, route: string): boolean =>
   middleware.some((middlewarePath) => route.startsWith(middlewarePath))
 
-const generateHiddenPathRedirects = ({ basePath }: Pick<NextConfig, 'basePath'>): NetlifyConfig['redirects'] =>
-  HIDDEN_PATHS.map((path) => ({
-    from: `${basePath}${path}`,
-    to: '/404.html',
-    status: 404,
-    force: true,
-  }))
-
 const generateLocaleRedirects = ({
   i18n,
   basePath,
@@ -281,8 +273,6 @@ export const generateRedirects = async ({
     join(netlifyConfig.build.publish, 'routes-manifest.json'),
   )
 
-  netlifyConfig.redirects.push(...generateHiddenPathRedirects({ basePath }))
-
   if (i18n && i18n.localeDetection !== false) {
     netlifyConfig.redirects.push(...generateLocaleRedirects({ i18n, basePath, trailingSlash }))
   }

packages/runtime/src/helpers/utils.ts

@@ -170,7 +170,7 @@ export const redirectsForNext404Route = ({
 }): NetlifyConfig['redirects'] =>
   netlifyRoutesForNextRoute({ route, buildId, i18n }).map(({ redirect, locale }) => ({
     from: `${basePath}${redirect}`,
-    to: locale ? `${basePath}/server/pages/${locale}/404.html` : `${basePath}/server/pages/404.html`,
+    to: locale ? `${basePath}/${locale}/404.html` : `${basePath}/404.html`,
     status: 404,
     force,
   }))

packages/runtime/src/index.ts

@@ -17,7 +17,7 @@ import {
 } from './helpers/config'
 import { onPreDev } from './helpers/dev'
 import { writeEdgeFunctions, loadMiddlewareManifest, cleanupEdgeFunctions } from './helpers/edge'
-import { moveStaticPages, movePublicFiles, patchNextFiles } from './helpers/files'
+import { moveStaticPages, movePublicFiles, patchNextFiles, removeMetadataFiles } from './helpers/files'
 import { splitApiRoutes } from './helpers/flags'
 import {
   generateFunctions,
@@ -254,6 +254,12 @@ const plugin: NetlifyPlugin = {
     warnForProblematicUserRewrites({ basePath, redirects })
     warnForRootRedirects({ appDir })
     await warnOnApiRoutes({ FUNCTIONS_DIST })
+
+    // we are removing metadata files from Publish directory
+    // we have to do this after functions were bundled as bundling still
+    // require those files, but we don't want to publish them
+    await removeMetadataFiles(publish)
+
     if (experimental?.appDir) {
       console.log(
         '🧪 Thank you for testing "appDir" support on Netlify. For known issues and to give feedback, visit https://ntl.fyi/next-13-feedback',