[v2] Hulksmash build slowdowns on larger sites

packages/gatsby-cli/package.json

@@ -18,6 +18,7 @@
     "core-js": "^2.5.0",
     "envinfo": "^5.8.1",
     "execa": "^0.8.0",
+    "fs-exists-cached": "^1.0.0",
     "fs-extra": "^4.0.1",
     "hosted-git-info": "^2.6.0",
     "lodash": "^4.17.4",

packages/gatsby-cli/src/create-cli.js

@@ -2,8 +2,8 @@ const path = require(`path`)
 const resolveCwd = require(`resolve-cwd`)
 const yargs = require(`yargs`)
 const report = require(`./reporter`)
-const fs = require(`fs`)
 const envinfo = require(`envinfo`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const DEFAULT_BROWSERS = [`>0.25%`, `not dead`]
 
@@ -19,7 +19,7 @@ function buildLocalCommands(cli, isLocalSite) {
   const directory = path.resolve(`.`)
 
   let siteInfo = { directory, browserslist: DEFAULT_BROWSERS }
-  const useYarn = fs.existsSync(path.join(directory, `yarn.lock`))
+  const useYarn = existsSync(path.join(directory, `yarn.lock`))
   if (isLocalSite) {
     const json = require(path.join(directory, `package.json`))
     siteInfo.sitePackageJson = json

packages/gatsby-cli/src/init-starter.js

@@ -6,6 +6,7 @@ const fs = require(`fs-extra`)
 const sysPath = require(`path`)
 const report = require(`./reporter`)
 const url = require(`url`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const spawn = (cmd: string) => {
   const [file, ...args] = cmd.split(/\s+/)
@@ -49,7 +50,7 @@ const copy = async (starterPath: string, rootPath: string) => {
   // 493 = parseInt('755', 8)
   await fs.mkdirp(rootPath, { mode: 493 })
 
-  if (!fs.existsSync(starterPath)) {
+  if (!existsSync(starterPath)) {
     throw new Error(`starter ${starterPath} doesn't exist`)
   }
 
@@ -117,7 +118,7 @@ module.exports = async (starter: string, options: InitOptions = {}) => {
     return
   }
 
-  if (fs.existsSync(sysPath.join(rootPath, `package.json`))) {
+  if (existsSync(sysPath.join(rootPath, `package.json`))) {
     report.panic(`Directory ${rootPath} is already an npm project`)
     return
   }

packages/gatsby-plugin-netlify/src/gatsby-node.js

@@ -34,7 +34,7 @@ exports.onPostBuild = async ({ store, pathPrefix }, userPluginOptions) => {
   let rewrites = []
   if (pluginOptions.generateMatchPathRewrites) {
     const { pages } = store.getState()
-    rewrites = pages
+    rewrites = Array.from(pages.values())
       .filter(page => page.matchPath && page.matchPath !== page.path)
       .map(page => {
         return {

packages/gatsby-plugin-page-creator/package.json

@@ -21,6 +21,7 @@
     "@babel/runtime": "7.0.0-beta.51",
     "bluebird": "^3.5.0",
     "chokidar": "^1.7.0",
+    "fs-exists-cached": "^1.0.0",
     "glob": "^7.1.1",
     "lodash": "^4.17.4",
     "parse-filepath": "^1.0.1",

packages/gatsby-plugin-page-creator/src/gatsby-node.js

@@ -3,7 +3,7 @@ const Promise = require(`bluebird`)
 const _ = require(`lodash`)
 const chokidar = require(`chokidar`)
 const systemPath = require(`path`)
-const fs = require(`fs`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const glob = Promise.promisify(globCB)
 
@@ -35,7 +35,7 @@ exports.createPagesStatefully = async (
   }
 
   // Validate that the path exists.
-  if (pathCheck && !fs.existsSync(pagesPath)) {
+  if (pathCheck && !existsSync(pagesPath)) {
     reporter.panic(
       `
       The path passed to gatsby-plugin-page-creator does not exist on your file system:

packages/gatsby-plugin-sharp/package.json

@@ -10,6 +10,7 @@
     "@babel/runtime": "7.0.0-beta.51",
     "async": "^2.1.2",
     "bluebird": "^3.5.0",
+    "fs-exists-cached": "^1.0.0",
     "imagemin": "^5.2.2",
     "imagemin-pngquant": "^5.1.0",
     "imagemin-webp": "^4.0.0",

packages/gatsby-plugin-sharp/src/index.js

@@ -10,6 +10,7 @@ const imageminPngquant = require(`imagemin-pngquant`)
 const imageminWebp = require(`imagemin-webp`)
 const queue = require(`async/queue`)
 const path = require(`path`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const imageSizeCache = new Map()
 const getImageSize = file => {
@@ -245,7 +246,7 @@ const queueJob = (job, reporter) => {
   }
 
   // Check if the output file already exists so we don't redo work.
-  if (fs.existsSync(job.outputPath)) {
+  if (existsSync(job.outputPath)) {
     return
   }
 

packages/gatsby-transformer-remark/src/extend-node-type.js

@@ -24,6 +24,8 @@ const remark2retext = require(`remark-retext`)
 const stripPosition = require(`unist-util-remove-position`)
 const hastReparseRaw = require(`hast-util-raw`)
 
+global.mdToHTML = []
+
 let pluginsCacheStr = ``
 let pathPrefixCacheStr = ``
 const astCacheKey = node =>
@@ -280,6 +282,7 @@ module.exports = (
       if (cachedHTML) {
         return cachedHTML
       } else {
+        const start = process.hrtime()
         const ast = await getHTMLAst(markdownNode)
         // Save new HTML to cache and return
         const html = hastToHTML(ast, {
@@ -288,6 +291,9 @@ module.exports = (
 
         // Save new HTML to cache and return
         cache.set(htmlCacheKey(markdownNode), html)
+        global.mdToHTML.push(
+          require(`convert-hrtime`)(process.hrtime(start)).milliseconds
+        )
         return html
       }
     }

packages/gatsby/cache-dir/static-entry.js

@@ -9,6 +9,10 @@ const apiRunner = require(`./api-runner-ssr`)
 const syncRequires = require(`./sync-requires`)
 const { dataPaths, pages } = require(`./data.json`)
 
+// Speed up looking up pages.
+const pagesObjectMap = new Map()
+pages.forEach(p => pagesObjectMap.set(p.path, p))
+
 const stats = JSON.parse(
   fs.readFileSync(`${process.cwd()}/public/webpack.stats.json`, `utf-8`)
 )
@@ -48,7 +52,7 @@ function urlJoin(...parts) {
   }, ``)
 }
 
-const getPage = path => pages.find(page => page.path === path)
+const getPage = path => pagesObjectMap.get(path)
 
 const createElement = React.createElement
 

packages/gatsby/package.json

@@ -70,6 +70,7 @@
     "graphql-relay": "^0.5.5",
     "graphql-skip-limit": "^2.0.0-beta.2",
     "graphql-type-json": "^0.2.1",
+    "hash-mod": "^0.0.5",
     "history": "^4.6.2",
     "invariant": "^2.2.4",
     "is-relative": "^1.0.0",
@@ -117,6 +118,7 @@
     "socket.io": "^2.0.3",
     "string-similarity": "^1.2.0",
     "style-loader": "^0.19.1",
+    "treeify": "^1.1.0",
     "type-of": "^2.0.1",
     "uglifyjs-webpack-plugin": "^1.2.4",
     "url-loader": "^1.0.1",

packages/gatsby/src/bootstrap/get-config-file.js

@@ -5,6 +5,7 @@ const testRequireError = require(`../utils/test-require-error`).default
 const report = require(`gatsby-cli/lib/reporter`)
 const chalk = require(`chalk`)
 const path = require(`path`)
+const existsSync = require(`fs-exists-cached`).sync
 
 function isNearMatch(
   fileName: string,
@@ -45,7 +46,7 @@ module.exports = async function getConfigFile(
       )
       console.log(``)
       process.exit(1)
-    } else if (fs.existsSync(path.join(rootDir, `src`, configName))) {
+    } else if (existsSync(path.join(rootDir, `src`, configName))) {
       console.log(``)
       report.error(
         `Your ${configName} file is in the wrong place. You've placed it in the src/ directory. It must instead be at the root of your site next to your package.json file.`

packages/gatsby/src/bootstrap/index.js

@@ -159,8 +159,13 @@ module.exports = async (args: BootstrapArgs) => {
   // directory.
   initCache()
 
-  // Ensure the public/static directory is created.
-  await fs.ensureDirSync(`${program.directory}/public/static/d`)
+  // Ensure the public/static directory and data subdirectories are created.
+  await fs.ensureDir(`${program.directory}/public/static`)
+  await Promise.all(
+    _.range(0, 999).map(i =>
+      fs.ensureDir(`${program.directory}/public/static/d/${i}`)
+    )
+  )
 
   // Copy our site files to the root of the site.
   activity = report.activityTimer(`copy gatsby files`)

packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.js.snap

@@ -36,7 +36,6 @@ Array [
     "name": "internal-data-bridge",
     "nodeAPIs": Array [
       "sourceNodes",
-      "onCreatePage",
     ],
     "pluginOptions": Object {
       "plugins": Array [],
@@ -140,7 +139,6 @@ Array [
     "name": "internal-data-bridge",
     "nodeAPIs": Array [
       "sourceNodes",
-      "onCreatePage",
     ],
     "pluginOptions": Object {
       "plugins": Array [],

packages/gatsby/src/bootstrap/load-plugins/load.js

@@ -5,6 +5,7 @@ const path = require(`path`)
 const crypto = require(`crypto`)
 const glob = require(`glob`)
 const { store } = require(`../../redux`)
+const existsSync = require(`fs-exists-cached`).sync
 
 function createFileContentHash(root, globPattern) {
   const hash = crypto.createHash(`md5`)
@@ -34,12 +35,12 @@ function createFileContentHash(root, globPattern) {
  */
 function resolvePlugin(pluginName) {
   // Only find plugins when we're not given an absolute path
-  if (!fs.existsSync(pluginName)) {
+  if (!existsSync(pluginName)) {
     // Find the plugin in the local plugins folder
     const resolvedPath = slash(path.resolve(`./plugins/${pluginName}`))
 
-    if (fs.existsSync(resolvedPath)) {
-      if (fs.existsSync(`${resolvedPath}/package.json`)) {
+    if (existsSync(resolvedPath)) {
+      if (existsSync(`${resolvedPath}/package.json`)) {
         const packageJSON = JSON.parse(
           fs.readFileSync(`${resolvedPath}/package.json`, `utf-8`)
         )

packages/gatsby/src/commands/build-html.js

@@ -13,7 +13,7 @@ module.exports = async (program: any) => {
 
   debug(`generating static HTML`)
   // Reduce pages objects to an array of paths.
-  const pages = store.getState().pages.map(page => page.path)
+  const pages = Array.from(store.getState().pages.values(), page => page.path)
 
   // Static site generation.
   const compilerConfig = await webpackConfig(

packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js

@@ -40,6 +40,7 @@ function transformPackageJson(json) {
   return json
 }
 
+const createPageId = path => `SitePage ${path}`
 exports.sourceNodes = ({ actions, store }) => {
   const { createNode } = actions
   const state = store.getState()
@@ -136,37 +137,3 @@ exports.sourceNodes = ({ actions, store }) => {
     }
   })
 }
-
-const createPageId = path => `SitePage ${path}`
-
-exports.onCreatePage = ({ page, actions }) => {
-  const { createNode } = actions
-  // eslint-disable-next-line
-  const { updatedAt, ...pageWithoutUpdated } = page
-
-  // Add page.
-  createNode({
-    ...pageWithoutUpdated,
-    id: createPageId(page.path),
-    parent: `SOURCE`,
-    children: [],
-    internal: {
-      type: `SitePage`,
-      contentDigest: crypto
-        .createHash(`md5`)
-        .update(JSON.stringify(page))
-        .digest(`hex`),
-      description:
-        page.pluginCreatorId === `Plugin default-site-plugin`
-          ? `Your site's "gatsby-node.js"`
-          : page.pluginCreatorId,
-    },
-  })
-}
-
-// Listen for DELETE_PAGE and delete page nodes.
-emitter.on(`DELETE_PAGE`, action => {
-  const nodeId = createPageId(action.payload.path)
-  const node = getNode(nodeId)
-  boundActionCreators.deleteNode({ node })
-})

packages/gatsby/src/internal-plugins/load-babel-config/gatsby-node.js

@@ -5,6 +5,7 @@ const path = require(`path`)
 const json5 = require(`json5`)
 const report = require(`gatsby-cli/lib/reporter`)
 const { actionifyBabelrc, addDefaultPluginsPresets } = require(`./utils`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const testRequireError = require(`../../utils/test-require-error`).default
 
@@ -15,7 +16,7 @@ const testRequireError = require(`../../utils/test-require-error`).default
  */
 function findBabelrc(directory) {
   const babelrcPath = path.join(directory, `.babelrc`)
-  if (fs.existsSync(babelrcPath)) {
+  if (existsSync(babelrcPath)) {
     try {
       const babelrc = fs.readFileSync(babelrcPath, `utf-8`)
       return json5.parse(babelrc)

packages/gatsby/src/internal-plugins/query-runner/gatsby-node.js

@@ -3,12 +3,21 @@ const { watchComponent } = require(`./query-watcher`)
 let components = {}
 
 exports.onCreatePage = ({ page, store }) => {
-  const component = store.getState().components.get(page.componentPath)
+  // In development, watch the component to detect query changes.
+  if (process.env.NODE_ENV !== `production`) {
+    const component = store.getState().components.get(page.componentPath)
 
-  if (components[component.componentPath]) {
-    return
-  }
+    if (!component) {
+      console.log({ components: store.getState().components })
+      console.log({ page })
+      console.log({ component })
+      process.exit()
+    }
+
+    if (components[component.componentPath]) {
+      return
+    }
 
-  // Watch the component to detect query changes.
-  watchComponent(component.componentPath)
+    watchComponent(component.componentPath)
+  }
 }