Add build manifest (#4119)

* Add build manifest

* Split out css since they don’t have exact name

* Remove pages map

* Fix locations test

* Re-run tests

* Get consistent open ports

* Fix static tests

* Add comment about Cache-Control header

bin/next-export

@@ -57,8 +57,6 @@ const options = {
   outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out')
 }
 
-exportApp(dir, options)
-  .catch((err) => {
-    console.error(err)
-    process.exit(1)
-  })
+exportApp(dir, options).catch((err) => {
+  printAndExit(err)
+})

lib/constants.js

@@ -3,3 +3,4 @@ export const PHASE_PRODUCTION_BUILD = 'phase-production-build'
 export const PHASE_PRODUCTION_SERVER = 'phase-production-server'
 export const PHASE_DEVELOPMENT_SERVER = 'phase-development-server'
 export const PAGES_MANIFEST = 'pages-manifest.json'
+export const BUILD_MANIFEST = 'build-manifest.json'

package.json

@@ -134,7 +134,6 @@
     "node-fetch": "1.7.3",
     "node-notifier": "5.1.2",
     "nyc": "11.2.1",
-    "portfinder": "1.0.13",
     "react": "16.2.0",
     "react-dom": "16.2.0",
     "rimraf": "2.6.2",

readme.md

@@ -943,9 +943,9 @@ import flush from 'styled-jsx/server'
 
 export default class MyDocument extends Document {
   static getInitialProps({ renderPage }) {
-    const { html, head, errorHtml, chunks } = renderPage()
+    const { html, head, errorHtml, chunks, buildManifest } = renderPage()
     const styles = flush()
-    return { html, head, errorHtml, chunks, styles }
+    return { html, head, errorHtml, chunks, styles, buildManifest }
   }
 
   render() {

server/build/plugins/build-manifest-plugin.js

@@ -0,0 +1,46 @@
+// @flow
+import { RawSource } from 'webpack-sources'
+import {BUILD_MANIFEST} from '../../../lib/constants'
+
+// This plugin creates a build-manifest.json for all assets that are being output
+// It has a mapping of "entry" filename to real filename. Because the real filename can be hashed in production
+export default class BuildManifestPlugin {
+  apply (compiler: any) {
+    compiler.plugin('emit', (compilation, callback) => {
+      const {chunks} = compilation
+      const assetMap = {pages: {}, css: []}
+
+      for (const chunk of chunks) {
+        if (!chunk.name || !chunk.files) {
+          continue
+        }
+
+        const files = []
+
+        for (const file of chunk.files) {
+          if (/\.map$/.test(file)) {
+            continue
+          }
+
+          if (/\.hot-update\.js$/.test(file)) {
+            continue
+          }
+
+          if (/\.css$/.exec(file)) {
+            assetMap.css.push(file)
+            continue
+          }
+
+          files.push(file)
+        }
+
+        if (files.length > 0) {
+          assetMap[chunk.name] = files
+        }
+      }
+
+      compilation.assets[BUILD_MANIFEST] = new RawSource(JSON.stringify(assetMap))
+      callback()
+    })
+  }
+}

server/build/webpack.js

@@ -12,6 +12,7 @@ import NextJsSsrImportPlugin from './plugins/nextjs-ssr-import'
 import DynamicChunksPlugin from './plugins/dynamic-chunks-plugin'
 import UnlinkFilePlugin from './plugins/unlink-file-plugin'
 import PagesManifestPlugin from './plugins/pages-manifest-plugin'
+import BuildManifestPlugin from './plugins/build-manifest-plugin'
 
 const presetItem = createConfigItem(require('./babel/preset'), {type: 'preset'})
 const hotLoaderItem = createConfigItem(require('react-hot-loader/babel'), {type: 'plugin'})
@@ -259,14 +260,15 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
       }),
       !dev && new webpack.optimize.ModuleConcatenationPlugin(),
       isServer && new PagesManifestPlugin(),
+      !isServer && new BuildManifestPlugin(),
       !isServer && new PagesPlugin(),
       !isServer && new DynamicChunksPlugin(),
       isServer && new NextJsSsrImportPlugin(),
       // In dev mode, we don't move anything to the commons bundle.
       // In production we move common modules into the existing main.js bundle
       !isServer && new webpack.optimize.CommonsChunkPlugin({
         name: 'main.js',
-        filename: 'main.js',
+        filename: dev ? 'static/commons/main.js' : 'static/commons/main-[chunkhash].js',
         minChunks (module, count) {
           // React and React DOM are used everywhere in Next.js. So they should always be common. Even in development mode, to speed up compilation.
           if (module.resource && module.resource.includes(`${sep}react-dom${sep}`) && count >= 0) {
@@ -297,8 +299,8 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
       }),
       // We use a manifest file in development to speed up HMR
       dev && !isServer && new webpack.optimize.CommonsChunkPlugin({
-        name: 'manifest',
-        filename: 'manifest.js'
+        name: 'manifest.js',
+        filename: dev ? 'static/commons/manifest.js' : 'static/commons/manifest-[chunkhash].js'
       })
     ].filter(Boolean)
   }

server/document.js

@@ -10,9 +10,9 @@ const Fragment = React.Fragment || function Fragment ({ children }) {
 
 export default class Document extends Component {
   static getInitialProps ({ renderPage }) {
-    const { html, head, errorHtml, chunks } = renderPage()
+    const { html, head, errorHtml, chunks, buildManifest } = renderPage()
     const styles = flush()
-    return { html, head, errorHtml, chunks, styles }
+    return { html, head, errorHtml, chunks, styles, buildManifest }
   }
 
   static childContextTypes = {
@@ -40,32 +40,33 @@ export class Head extends Component {
   }
 
   getChunkPreloadLink (filename) {
-    const { __NEXT_DATA__ } = this.context._documentProps
+    const { __NEXT_DATA__, buildManifest } = this.context._documentProps
     let { assetPrefix, buildId } = __NEXT_DATA__
-    const hash = buildId
 
-    return (
-      <link
+    const files = buildManifest[filename]
+
+    return files.map(file => {
+      return <link
         key={filename}
         rel='preload'
-        href={`${assetPrefix}/_next/${hash}/${filename}`}
+        href={`${assetPrefix}/_next/${file}`}
         as='script'
       />
-    )
+    })
   }
 
   getPreloadMainLinks () {
     const { dev } = this.context._documentProps
     if (dev) {
       return [
-        this.getChunkPreloadLink('manifest.js'),
-        this.getChunkPreloadLink('main.js')
+        ...this.getChunkPreloadLink('manifest.js'),
+        ...this.getChunkPreloadLink('main.js')
       ]
     }
 
     // In the production mode, we have a single asset with all the JS content.
     return [
-      this.getChunkPreloadLink('main.js')
+      ...this.getChunkPreloadLink('main.js')
     ]
   }
 
@@ -125,31 +126,32 @@ export class NextScript extends Component {
   }
 
   getChunkScript (filename, additionalProps = {}) {
-    const { __NEXT_DATA__ } = this.context._documentProps
+    const { __NEXT_DATA__, buildManifest } = this.context._documentProps
     let { assetPrefix, buildId } = __NEXT_DATA__
-    const hash = buildId
 
-    return (
+    const files = buildManifest[filename]
+
+    return files.map((file) => (
       <script
         key={filename}
-        src={`${assetPrefix}/_next/${hash}/${filename}`}
+        src={`${assetPrefix}/_next/${file}`}
         {...additionalProps}
       />
-    )
+    ))
   }
 
   getScripts () {
     const { dev } = this.context._documentProps
     if (dev) {
       return [
-        this.getChunkScript('manifest.js'),
-        this.getChunkScript('main.js')
+        ...this.getChunkScript('manifest.js'),
+        ...this.getChunkScript('main.js')
       ]
     }
 
     // In the production mode, we have a single asset with all the JS content.
     // So, we can load the script with async
-    return [this.getChunkScript('main.js', { async: true })]
+    return [...this.getChunkScript('main.js', { async: true })]
   }
 
   getDynamicChunks () {

server/export.js

@@ -19,10 +19,7 @@ export default async function (dir, options, configuration) {
   log(`> using build directory: ${nextDir}`)
 
   if (!existsSync(nextDir)) {
-    console.error(
-      `Build directory ${nextDir} does not exist. Make sure you run "next build" before running "next start" or "next export".`
-    )
-    process.exit(1)
+    throw new Error(`Build directory ${nextDir} does not exist. Make sure you run "next build" before running "next start" or "next export".`)
   }
 
   const buildId = readFileSync(join(nextDir, 'BUILD_ID'), 'utf8')
@@ -53,12 +50,6 @@ export default async function (dir, options, configuration) {
     )
   }
 
-  // Copy main.js
-  await cp(
-    join(nextDir, 'main.js'),
-    join(outDir, '_next', buildId, 'main.js')
-  )
-
   // Copy .next/static directory
   if (existsSync(join(nextDir, 'static'))) {
     log('  copying "static build" directory')

server/index.js

@@ -162,57 +162,6 @@ export default class Server {
         await this.serveStatic(req, res, p)
       },
 
-      '/_next/:buildId/manifest.js': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildId(params.buildId, res)
-        const p = join(this.dir, this.dist, 'manifest.js')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:buildId/manifest.js.map': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildId(params.buildId, res)
-        const p = join(this.dir, this.dist, 'manifest.js.map')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:buildId/main.js': async (req, res, params) => {
-        if (this.dev) {
-          this.handleBuildId(params.buildId, res)
-          const p = join(this.dir, this.dist, 'main.js')
-          await this.serveStatic(req, res, p)
-        } else {
-          const buildId = params.buildId
-          if (!this.handleBuildId(buildId, res)) {
-            const error = new Error('INVALID_BUILD_ID')
-            const customFields = { buildIdMismatched: true }
-
-            return await renderScriptError(req, res, '/_error', error, customFields, this.renderOpts)
-          }
-
-          const p = join(this.dir, this.dist, 'main.js')
-          await this.serveStatic(req, res, p)
-        }
-      },
-
-      '/_next/:buildId/main.js.map': async (req, res, params) => {
-        if (this.dev) {
-          this.handleBuildId(params.buildId, res)
-          const p = join(this.dir, this.dist, 'main.js.map')
-          await this.serveStatic(req, res, p)
-        } else {
-          const buildId = params.buildId
-          if (!this.handleBuildId(buildId, res)) {
-            return await this.render404(req, res)
-          }
-
-          const p = join(this.dir, this.dist, 'main.js.map')
-          await this.serveStatic(req, res, p)
-        }
-      },
-
       '/_next/:buildId/page/:path*.js.map': async (req, res, params) => {
         const paths = params.path || ['']
         const page = `/${paths.join('/')}`
@@ -279,6 +228,11 @@ export default class Server {
       },
 
       '/_next/static/:path*': async (req, res, params) => {
+        // The commons folder holds commonschunk files
+        // In development they don't have a hash, and shouldn't be cached by the browser.
+        if (this.dev && params.path[0] === 'commons') {
+          res.setHeader('Cache-Control', 'no-store, must-revalidate')
+        }
         const p = join(this.dir, this.dist, 'static', ...(params.path || []))
         await this.serveStatic(req, res, p)
       },

server/render.js

@@ -12,6 +12,7 @@ import Head, { defaultHead } from '../lib/head'
 import App from '../lib/app'
 import ErrorDebug from '../lib/error-debug'
 import { flushChunks } from '../lib/dynamic'
+import { BUILD_MANIFEST } from '../lib/constants'
 
 const logger = console
 
@@ -54,6 +55,7 @@ async function doRender (req, res, pathname, query, {
   }
 
   const documentPath = join(dir, dist, 'dist', 'bundles', 'pages', '_document')
+  const buildManifest = require(join(dir, dist, BUILD_MANIFEST))
 
   let [Component, Document] = await Promise.all([
     requirePage(page, {dir, dist}),
@@ -94,7 +96,7 @@ async function doRender (req, res, pathname, query, {
     }
     const chunks = loadChunks({ dev, dir, dist, availableChunks })
 
-    return { html, head, errorHtml, chunks }
+    return { html, head, errorHtml, chunks, buildManifest }
   }
 
   const docProps = await loadGetInitialProps(Document, { ...ctx, renderPage })
@@ -117,6 +119,7 @@ async function doRender (req, res, pathname, query, {
     dev,
     dir,
     staticMarkup,
+    buildManifest,
     ...docProps
   })
 

test/integration/dist-dir/test/index.test.js

@@ -39,11 +39,10 @@ describe('Production Usage', () => {
 
   describe('File locations', () => {
     it('should build the app within the given `dist` directory', () => {
-      expect(existsSync(join(__dirname, '/../dist/main.js'))).toBeTruthy()
+      expect(existsSync(join(__dirname, '/../dist/BUILD_ID'))).toBeTruthy()
     })
-
     it('should not build the app within the default `.next` directory', () => {
-      expect(existsSync(join(__dirname, '/../.next/main.js'))).toBeFalsy()
+      expect(existsSync(join(__dirname, '/../.next/BUILD_ID'))).toBeFalsy()
     })
   })
 })

test/integration/static/.gitignore

@@ -0,0 +1 @@
+.next-dev

test/integration/static/next.config.js

@@ -1,17 +1,22 @@
-module.exports = {
-  exportPathMap: function () {
-    return {
-      '/': { page: '/' },
-      '/about': { page: '/about' },
-      '/asset': { page: '/asset' },
-      '/button-link': { page: '/button-link' },
-      '/get-initial-props-with-no-query': { page: '/get-initial-props-with-no-query' },
-      '/counter': { page: '/counter' },
-      '/dynamic-imports': { page: '/dynamic-imports' },
-      '/dynamic': { page: '/dynamic', query: { text: 'cool dynamic text' } },
-      '/dynamic/one': { page: '/dynamic', query: { text: 'next export is nice' } },
-      '/dynamic/two': { page: '/dynamic', query: { text: 'zeit is awesome' } },
-      '/file-name.md': { page: '/dynamic', query: { text: 'this file has an extension' } }
+const {PHASE_DEVELOPMENT_SERVER} = require('next/constants')
+
+module.exports = (phase) => {
+  return {
+    distDir: phase === PHASE_DEVELOPMENT_SERVER ? '.next-dev' : '.next',
+    exportPathMap: function () {
+      return {
+        '/': { page: '/' },
+        '/about': { page: '/about' },
+        '/asset': { page: '/asset' },
+        '/button-link': { page: '/button-link' },
+        '/get-initial-props-with-no-query': { page: '/get-initial-props-with-no-query' },
+        '/counter': { page: '/counter' },
+        '/dynamic-imports': { page: '/dynamic-imports' },
+        '/dynamic': { page: '/dynamic', query: { text: 'cool dynamic text' } },
+        '/dynamic/one': { page: '/dynamic', query: { text: 'next export is nice' } },
+        '/dynamic/two': { page: '/dynamic', query: { text: 'zeit is awesome' } },
+        '/file-name.md': { page: '/dynamic', query: { text: 'this file has an extension' } }
+      }
     }
   }
 }