Add flow, pages-manifest.json, defaultPathMap for export (minor) (#4066)

* Initial implementation of next export without exportPathMap

* Shorter message

* Set up flow

* Create pages manifest

* Use pagesManifest for next export

* Fix tests

* Document defaultPathMap

* Replacing the path is no longer needed

* Use posix normalize for consistent behaviour

* Remove second instance of examples

* Add comment about what pages-manifest does

* Make windows path a route

.babelrc

@@ -1,7 +1,8 @@
 {
   "presets": [
     "env",
-    "react"
+    "react",
+    "flow"
   ],
   "plugins": [
     "transform-object-rest-spread",

.flowconfig

@@ -0,0 +1,2 @@
+[ignore]
+<PROJECT_ROOT>/examples/.*

lib/constants.js

@@ -2,3 +2,4 @@ export const PHASE_EXPORT = 'phase-export'
 export const PHASE_PRODUCTION_BUILD = 'phase-production-build'
 export const PHASE_PRODUCTION_SERVER = 'phase-production-server'
 export const PHASE_DEVELOPMENT_SERVER = 'phase-development-server'
+export const PAGES_MANIFEST = 'pages-manifest.json'

package.json

@@ -118,6 +118,7 @@
     "babel-plugin-istanbul": "4.1.5",
     "babel-plugin-transform-remove-strict-mode": "0.0.2",
     "babel-preset-es2015": "6.24.1",
+    "babel-preset-flow": "6.23.0",
     "benchmark": "2.1.4",
     "cheerio": "0.22.0",
     "chromedriver": "2.32.3",

readme.md

@@ -1259,7 +1259,7 @@ Simply develop your app as you normally do with Next.js. Then create a custom Ne
 ```js
 // next.config.js
 module.exports = {
-  exportPathMap: function() {
+  exportPathMap: function(defaultPathMap) {
     return {
       '/': { page: '/' },
       '/about': { page: '/about' },

server/build/plugins/pages-manifest-plugin.js

@@ -0,0 +1,30 @@
+// @flow
+import { RawSource } from 'webpack-sources'
+import { MATCH_ROUTE_NAME } from '../../utils'
+import {PAGES_MANIFEST} from '../../../lib/constants'
+
+// This plugin creates a pages-manifest.json from page entrypoints.
+// This is used for mapping paths like `/` to `.next/dist/bundles/pages/index.js` when doing SSR
+// It's also used by next export to provide defaultPathMap
+export default class PagesManifestPlugin {
+  apply (compiler: any) {
+    compiler.plugin('emit', (compilation, callback) => {
+      const {entries} = compilation
+      const pages = {}
+
+      for (const entry of entries) {
+        const pagePath = MATCH_ROUTE_NAME.exec(entry.name)[1]
+
+        if (!pagePath) {
+          continue
+        }
+
+        const {name} = entry
+        pages[`/${pagePath.replace(/\\/g, '/')}`] = name
+      }
+
+      compilation.assets[PAGES_MANIFEST] = new RawSource(JSON.stringify(pages))
+      callback()
+    })
+  }
+}

server/build/webpack.js

@@ -10,6 +10,7 @@ import PagesPlugin from './plugins/pages-plugin'
 import NextJsSsrImportPlugin from './plugins/nextjs-ssr-import'
 import DynamicChunksPlugin from './plugins/dynamic-chunks-plugin'
 import UnlinkFilePlugin from './plugins/unlink-file-plugin'
+import PagesManifestPlugin from './plugins/pages-manifest-plugin'
 import findBabelConfig from './babel/find-config'
 
 const nextDir = path.join(__dirname, '..', '..', '..')
@@ -254,6 +255,7 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
         'process.env.NODE_ENV': JSON.stringify(dev ? 'development' : 'production')
       }),
       !dev && new webpack.optimize.ModuleConcatenationPlugin(),
+      isServer && new PagesManifestPlugin(),
       !isServer && new PagesPlugin(),
       !isServer && new DynamicChunksPlugin(),
       isServer && new NextJsSsrImportPlugin(),

server/build/webpack/utils.js

@@ -9,7 +9,7 @@ export async function getPages (dir, {dev, isServer, pageExtensions}) {
   return getPageEntries(pageFiles, {isServer, pageExtensions})
 }
 
-async function getPagePaths (dir, {dev, isServer, pageExtensions}) {
+export async function getPagePaths (dir, {dev, isServer, pageExtensions}) {
   let pages
 
   if (dev) {

server/config.js

@@ -1,3 +1,4 @@
+// @flow
 import findUp from 'find-up'
 
 const cache = new Map()
@@ -11,28 +12,29 @@ const defaultConfig = {
   configOrigin: 'default',
   useFileSystemPublicRoutes: true,
   generateEtags: true,
-  pageExtensions: ['jsx', 'js'] // jsx before js because otherwise regex matching will match js first
+  pageExtensions: ['jsx', 'js']
 }
 
-export default function getConfig (phase, dir, customConfig) {
+export default function getConfig (phase: string, dir: string, customConfig?: ?Object) {
   if (!cache.has(dir)) {
     cache.set(dir, loadConfig(phase, dir, customConfig))
   }
   return cache.get(dir)
 }
 
-export function loadConfig (phase, dir, customConfig) {
+export function loadConfig (phase: string, dir: string, customConfig?: ?Object) {
   if (customConfig && typeof customConfig === 'object') {
     customConfig.configOrigin = 'server'
     return withDefaults(customConfig)
   }
-  const path = findUp.sync('next.config.js', {
+  const path: string = findUp.sync('next.config.js', {
     cwd: dir
   })
 
   let userConfig = {}
 
   if (path && path.length) {
+    // $FlowFixMe
     const userConfigModule = require(path)
     userConfig = userConfigModule.default || userConfigModule
     if (typeof userConfigModule === 'function') {
@@ -44,6 +46,6 @@ export function loadConfig (phase, dir, customConfig) {
   return withDefaults(userConfig)
 }
 
-function withDefaults (config) {
+function withDefaults (config: Object) {
   return Object.assign({}, defaultConfig, config)
 }

server/export.js

@@ -5,10 +5,9 @@ import walk from 'walk'
 import { extname, resolve, join, dirname, sep } from 'path'
 import { existsSync, readFileSync, writeFileSync } from 'fs'
 import getConfig from './config'
-import {PHASE_EXPORT} from '../lib/constants'
+import {PHASE_EXPORT, PAGES_MANIFEST} from '../lib/constants'
 import { renderToHTML } from './render'
 import { getAvailableChunks } from './utils'
-import { printAndExit } from '../lib/utils'
 import { setAssetPrefix } from '../lib/asset'
 import * as envConfig from '../lib/runtime-config'
 
@@ -17,7 +16,7 @@ export default async function (dir, options, configuration) {
   const nextConfig = configuration || getConfig(PHASE_EXPORT, dir)
   const nextDir = join(dir, nextConfig.distDir)
 
-  log(`  using build directory: ${nextDir}`)
+  log(`> using build directory: ${nextDir}`)
 
   if (!existsSync(nextDir)) {
     console.error(
@@ -27,6 +26,17 @@ export default async function (dir, options, configuration) {
   }
 
   const buildId = readFileSync(join(nextDir, 'BUILD_ID'), 'utf8')
+  const pagesManifest = require(join(nextDir, 'dist', PAGES_MANIFEST))
+
+  const pages = Object.keys(pagesManifest)
+  const defaultPathMap = {}
+
+  for (const page of pages) {
+    if (page === '/_document') {
+      continue
+    }
+    defaultPathMap[page] = { page }
+  }
 
   // Initialize the output directory
   const outDir = options.outdir
@@ -73,13 +83,13 @@ export default async function (dir, options, configuration) {
 
   // Get the exportPathMap from the `next.config.js`
   if (typeof nextConfig.exportPathMap !== 'function') {
-    printAndExit(
-      '> Could not find "exportPathMap" function inside "next.config.js"\n' +
-      '> "next export" uses that function to build html pages.'
-    )
+    console.log('> No "exportPathMap" found in "next.config.js". Generating map from "./pages"')
+    nextConfig.exportPathMap = async (defaultMap) => {
+      return defaultMap
+    }
   }
 
-  const exportPathMap = await nextConfig.exportPathMap()
+  const exportPathMap = await nextConfig.exportPathMap(defaultPathMap)
   const exportPaths = Object.keys(exportPathMap)
 
   // Start the rendering process
@@ -115,7 +125,7 @@ export default async function (dir, options, configuration) {
   }
 
   for (const path of exportPaths) {
-    log(`  exporting path: ${path}`)
+    log(`> exporting path: ${path}`)
     if (!path.startsWith('/')) {
       throw new Error(`path "${path}" doesn't start with a backslash`)
     }

server/require.js

@@ -1,5 +1,5 @@
-import {join, parse, normalize, sep} from 'path'
-import fs from 'mz/fs'
+import {join, posix} from 'path'
+import {PAGES_MANIFEST} from '../lib/constants'
 
 export function pageNotFoundError (page) {
   const err = new Error(`Cannot find module for page: ${page}`)
@@ -18,13 +18,8 @@ export function normalizePagePath (page) {
     page = `/${page}`
   }
 
-  // Windows compatibility
-  if (sep !== '/') {
-    page = page.replace(/\//g, sep)
-  }
-
   // Throw when using ../ etc in the pathname
-  const resolvedPage = normalize(page)
+  const resolvedPage = posix.normalize(page)
   if (page !== resolvedPage) {
     throw new Error('Requested and resolved page mismatch')
   }
@@ -33,7 +28,8 @@ export function normalizePagePath (page) {
 }
 
 export function getPagePath (page, {dir, dist}) {
-  const pageBundlesPath = join(dir, dist, 'dist', 'bundles', 'pages')
+  const serverBuildPath = join(dir, dist, 'dist')
+  const pagesManifest = require(join(serverBuildPath, PAGES_MANIFEST))
 
   try {
     page = normalizePagePath(page)
@@ -42,24 +38,14 @@ export function getPagePath (page, {dir, dist}) {
     throw pageNotFoundError(page)
   }
 
-  const pagePath = join(pageBundlesPath, page) // Path to the page that is to be loaded
-
-  // Don't allow wandering outside of the bundles directory
-  const pathDir = parse(pagePath).dir
-  if (pathDir.indexOf(pageBundlesPath) !== 0) {
-    console.error('Resolved page path goes outside of bundles path')
+  if (!pagesManifest[page]) {
     throw pageNotFoundError(page)
   }
 
-  return pagePath
+  return join(serverBuildPath, pagesManifest[page])
 }
 
 export default async function requirePage (page, {dir, dist}) {
-  const pagePath = getPagePath(page, {dir, dist}) + '.js'
-  const fileExists = await fs.exists(pagePath)
-  if (!fileExists) {
-    throw pageNotFoundError(page)
-  }
-
+  const pagePath = getPagePath(page, {dir, dist})
   return require(pagePath)
 }

test/isolated/_resolvedata/dist/bundles/pages/_error.js

@@ -0,0 +1,3 @@
+module.exports = {
+  test: 'error'
+}

test/isolated/_resolvedata/dist/pages-manifest.json

@@ -0,0 +1,6 @@
+{
+  "/index": "bundles/pages/index.js",
+  "/world": "bundles/pages/world.js",
+  "/_error": "bundles/pages/_error.js",
+  "/non-existent-child": "bundles/pages/non-existent-child.js"
+}

test/isolated/require-page.test.js

@@ -1,12 +1,10 @@
 /* global describe, it, expect */
 
-import { join, sep } from 'path'
+import { join } from 'path'
 import requirePage, {getPagePath, normalizePagePath, pageNotFoundError} from '../../dist/server/require'
 
-const dir = '/path/to/some/project'
-const dist = '.next'
-
-const pathToBundles = join(dir, dist, 'dist', 'bundles', 'pages')
+const sep = '/'
+const pathToBundles = join(__dirname, '_resolvedata', 'dist', 'bundles', 'pages')
 
 describe('pageNotFoundError', () => {
   it('Should throw error with ENOENT code', () => {
@@ -42,17 +40,17 @@ describe('normalizePagePath', () => {
 
 describe('getPagePath', () => {
   it('Should append /index to the / page', () => {
-    const pagePath = getPagePath('/', {dir, dist})
-    expect(pagePath).toBe(join(pathToBundles, `${sep}index`))
+    const pagePath = getPagePath('/', {dir: __dirname, dist: '_resolvedata'})
+    expect(pagePath).toBe(join(pathToBundles, `${sep}index.js`))
   })
 
   it('Should prepend / when a page does not have it', () => {
-    const pagePath = getPagePath('_error', {dir, dist})
-    expect(pagePath).toBe(join(pathToBundles, `${sep}_error`))
+    const pagePath = getPagePath('_error', {dir: __dirname, dist: '_resolvedata'})
+    expect(pagePath).toBe(join(pathToBundles, `${sep}_error.js`))
   })
 
   it('Should throw with paths containing ../', () => {
-    expect(() => getPagePath('/../../package.json', {dir, dist})).toThrow()
+    expect(() => getPagePath('/../../package.json', {dir: __dirname, dist: '_resolvedata'})).toThrow()
   })
 })
 

yarn.lock

@@ -464,8 +464,8 @@ asynckit@^0.4.0:
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
 
 atob@^2.0.0:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/atob/-/atob-2.0.3.tgz#19c7a760473774468f20b2d2d03372ad7d4cbf5d"
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.0.tgz#ab2b150e51d7b122b9efc8d7340c06b6c41076bc"
 
 autoprefixer@^6.3.1:
   version "6.7.7"
@@ -1058,7 +1058,7 @@ babel-preset-es2015@6.24.1:
     babel-plugin-transform-es2015-unicode-regex "^6.24.1"
     babel-plugin-transform-regenerator "^6.24.1"
 
-babel-preset-flow@^6.23.0:
+babel-preset-flow@6.23.0, babel-preset-flow@^6.23.0:
   version "6.23.0"
   resolved "https://registry.yarnpkg.com/babel-preset-flow/-/babel-preset-flow-6.23.0.tgz#e71218887085ae9a24b5be4169affb599816c49d"
   dependencies:
@@ -1464,12 +1464,12 @@ caniuse-api@^1.5.2:
     lodash.uniq "^4.5.0"
 
 caniuse-db@^1.0.30000529, caniuse-db@^1.0.30000634, caniuse-db@^1.0.30000639:
-  version "1.0.30000820"
-  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000820.tgz#7c20e25cea1768b261b724f82e3a6a253aaa1468"
+  version "1.0.30000821"
+  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000821.tgz#3fcdc67c446a94a9cdd848248a4e3e54b2da7419"
 
 caniuse-lite@^1.0.30000792:
-  version "1.0.30000820"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000820.tgz#6e36ee75187a2c83d26d6504a1af47cc580324d2"
+  version "1.0.30000821"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000821.tgz#0f3223f1e048ed96451c56ca6cf197058c42cb93"
 
 capture-stack-trace@^1.0.0:
   version "1.0.0"
@@ -2373,8 +2373,8 @@ ee-first@1.1.1:
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
 
 electron-to-chromium@^1.2.7, electron-to-chromium@^1.3.30:
-  version "1.3.40"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.40.tgz#1fbd6d97befd72b8a6f921dc38d22413d2f6fddf"
+  version "1.3.41"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.41.tgz#7e33643e00cd85edfd17e04194f6d00e73737235"
 
 elegant-spinner@^1.0.1:
   version "1.0.1"
@@ -2466,8 +2466,8 @@ es-to-primitive@^1.1.1:
     is-symbol "^1.0.1"
 
 es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14:
-  version "0.10.41"
-  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.41.tgz#bab3e982d750f0112f0cb9e6abed72c59eb33eb2"
+  version "0.10.42"
+  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.42.tgz#8c07dd33af04d5dcd1310b5cef13bea63a89ba8d"
   dependencies:
     es6-iterator "~2.0.3"
     es6-symbol "~3.1.1"
@@ -6870,8 +6870,8 @@ static-extend@^0.1.1:
     object-copy "^0.1.0"
 
 "statuses@>= 1.3.1 < 2":
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087"
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
 
 statuses@~1.3.1:
   version "1.3.1"