feat(gatsby): lazy bundle page components in dev server

packages/gatsby/cache-dir/__tests__/dev-loader.js

@@ -279,7 +279,7 @@ describe(`Dev loader`, () => {
   describe(`loadPage`, () => {
     const createSyncRequires = components => {
       return {
-        components,
+        lazyComponents: components,
       }
     }
 
@@ -375,29 +375,6 @@ describe(`Dev loader`, () => {
       })
     })
 
-    it(`should return an error when component cannot be loaded`, async () => {
-      const syncRequires = createSyncRequires({
-        chunk: false,
-      })
-      const devLoader = new DevLoader(syncRequires, [])
-      const pageData = {
-        path: `/mypage/`,
-        componentChunkName: `chunk`,
-        staticQueryHashes: [],
-      }
-      devLoader.loadPageDataJson = jest.fn(() =>
-        Promise.resolve({
-          payload: pageData,
-          status: `success`,
-        })
-      )
-
-      await devLoader.loadPage(`/mypage/`)
-      const expectation = devLoader.pageDb.get(`/mypage`)
-      expect(expectation).toHaveProperty(`status`, `error`)
-      expect(emitter.emit).toHaveBeenCalledTimes(0)
-    })
-
     it(`should return an error pageData contains an error`, async () => {
       const syncRequires = createSyncRequires({
         chunk: `instance`,

packages/gatsby/cache-dir/app.js

@@ -8,13 +8,15 @@ import emitter from "./emitter"
 import { apiRunner, apiRunnerAsync } from "./api-runner-browser"
 import { setLoader, publicLoader } from "./loader"
 import DevLoader from "./dev-loader"
-import syncRequires from "$virtual/sync-requires"
+import lazySyncRequires from "$virtual/lazy-client-sync-requires"
 // Generated during bootstrap
 import matchPaths from "$virtual/match-paths.json"
 
 window.___emitter = emitter
 
-const loader = new DevLoader(syncRequires, matchPaths)
+window.lazySyncRequires = lazySyncRequires
+
+const loader = new DevLoader(lazySyncRequires, matchPaths)
 setLoader(loader)
 loader.setApiRunner(apiRunner)
 

packages/gatsby/cache-dir/dev-loader.js

@@ -2,9 +2,41 @@ import { BaseLoader, PageResourceStatus } from "./loader"
 import { findPath } from "./find-path"
 
 class DevLoader extends BaseLoader {
-  constructor(syncRequires, matchPaths) {
-    const loadComponent = chunkName =>
-      Promise.resolve(syncRequires.components[chunkName])
+  constructor(lazyRequires, matchPaths) {
+    // One of the tests doesn't set a path.
+    const loadComponent = chunkName => {
+      if (process.env.NODE_ENV !== `test`) {
+        delete require.cache[
+          require.resolve(`$virtual/lazy-client-sync-requires`)
+        ]
+        lazyRequires = require(`$virtual/lazy-client-sync-requires`)
+      }
+      if (lazyRequires.lazyComponents[chunkName]) {
+        return Promise.resolve(lazyRequires.lazyComponents[chunkName])
+      } else {
+        return new Promise(resolve => {
+          // Tell the server the user wants to visit this page
+          // to trigger it compiling the page component's code.
+          const req = new XMLHttpRequest()
+          req.open(`post`, `/___client-page-visited`, true)
+          req.setRequestHeader(`Content-Type`, `application/json;charset=UTF-8`)
+          req.send(JSON.stringify({ chunkName }))
+
+          const checkForUpdates = setInterval(() => {
+            if (process.env.NODE_ENV !== `test`) {
+              delete require.cache[
+                require.resolve(`$virtual/lazy-client-sync-requires`)
+              ]
+            }
+            const lazyRequires = require(`$virtual/lazy-client-sync-requires`)
+            if (lazyRequires.lazyComponents[chunkName]) {
+              clearInterval(checkForUpdates)
+              resolve(lazyRequires.lazyComponents[chunkName])
+            }
+          }, 100)
+        })
+      }
+    }
     super(loadComponent, matchPaths)
   }
 

packages/gatsby/cache-dir/navigation.js

@@ -113,7 +113,6 @@ const navigate = (to, options = {}) => {
           })
         }
 
-        console.log(`Site has changed on server. Reloading browser`)
         window.location = pathname
       }
     }

packages/gatsby/src/bootstrap/__tests__/requires-writer.js

@@ -30,7 +30,8 @@ describe(`requires-writer`, () => {
   const program = {
     directory: `/dir`,
   }
-  let originalDateNow = global.Date.now
+  const clientVisitedPages = new Set()
+  const originalDateNow = global.Date.now
 
   beforeEach(() => {
     global.Date.now = () => now
@@ -61,6 +62,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       expect(spy).toBeCalledWith(
@@ -108,6 +110,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/login/`).path)
@@ -141,6 +144,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/clients/static`).path)
@@ -165,6 +169,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/`).path)
@@ -219,6 +224,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       expect(matchPaths.map(p => p.matchPath)).toMatchInlineSnapshot(`
@@ -245,13 +251,15 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       const matchPathsForInvertedInput = matchPaths
       pages = generatePagesState(pagesInput)
       await requiresWriter.writeAll({
         pages,
         program,
+        clientVisitedPages,
       })
 
       expect(matchPathsForInvertedInput).toEqual(matchPaths)
@@ -263,6 +271,7 @@ describe(`requires-writer`, () => {
         await requiresWriter.writeAll({
           pages,
           program,
+          clientVisitedPages,
         })
 
         const allMatchingPages = matchPaths

packages/gatsby/src/bootstrap/requires-writer.ts

@@ -162,26 +162,41 @@ const getMatchPaths = (
 
 const createHash = (
   matchPaths: Array<IGatsbyPageMatchPath>,
-  components: Array<IGatsbyPageComponent>
+  components: Array<IGatsbyPageComponent>,
+  cleanedClientVisitedPageComponents: Array<IGatsbyPageComponent>
 ): string =>
   crypto
     .createHash(`md5`)
-    .update(JSON.stringify({ matchPaths, components }))
+    .update(
+      JSON.stringify({
+        matchPaths,
+        components,
+        cleanedClientVisitedPageComponents,
+      })
+    )
     .digest(`hex`)
 
 // Write out pages information.
 export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
-  // console.log(`on requiresWriter progress`)
   const { program } = state
   const pages = [...state.pages.values()]
   const matchPaths = getMatchPaths(pages)
   const components = getComponents(pages)
+  const clientVisitedPageComponents = [...state.clientVisitedPages.values()]
+
+  // Remove any page components that no longer exist.
+  const cleanedClientVisitedPageComponents = components.filter(c =>
+    clientVisitedPageComponents.some(s => s === c.componentChunkName)
+  )
 
-  const newHash = createHash(matchPaths, components)
+  const newHash = createHash(
+    matchPaths,
+    components,
+    cleanedClientVisitedPageComponents
+  )
 
   if (newHash === lastHash) {
     // Nothing changed. No need to rewrite files
-    // console.log(`on requiresWriter END1`)
     return false
   }
 
@@ -211,6 +226,23 @@ const preferDefault = m => (m && m.default) || m
     .join(`,\n`)}
 }\n\n`
 
+  // Create file with sync requires of visited page components files.
+  let lazyClientSyncRequires = `${hotImport}
+  // prefer default export if available
+  const preferDefault = m => (m && m.default) || m
+  \n\n`
+  lazyClientSyncRequires += `exports.lazyComponents = {\n${cleanedClientVisitedPageComponents
+    .map(
+      (c: IGatsbyPageComponent): string =>
+        `  "${
+          c.componentChunkName
+        }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
+    )
+    .join(`,\n`)}
+  }\n\n`
+
+  writeModule(`$virtual/lazy-client-sync-requires`, lazyClientSyncRequires)
+
   // Create file with async requires of components/json files.
   let asyncRequires = `// prefer default export if available
 const preferDefault = m => (m && m.default) || m
@@ -264,45 +296,11 @@ const preferDefault = m => (m && m.default) || m
   return true
 }
 
-const debouncedWriteAll = _.debounce(
-  async (): Promise<void> => {
-    const activity = reporter.activityTimer(`write out requires`, {
-      id: `requires-writer`,
-    })
-    activity.start()
-    await writeAll(store.getState())
-    activity.end()
-  },
-  500,
-  {
-    // using "leading" can cause double `writeAll` call - particularly
-    // when refreshing data using `/__refresh` hook.
-    leading: false,
-  }
-)
-
 /**
- * Start listening to CREATE/DELETE_PAGE events so we can rewrite
+ * Start listening to CREATE_CLIENT_VISITED_PAGE events so we can rewrite
  * files as required
  */
-export const startListener = (): void => {
-  emitter.on(`CREATE_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`CREATE_PAGE_END`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`DELETE_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`DELETE_PAGE_BY_PATH`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-}
+emitter.on(`CREATE_CLIENT_VISITED_PAGE`, (): void => {
+  reporter.pendingActivity({ id: `requires-writer` })
+  writeAll(store.getState())
+})

packages/gatsby/src/redux/actions/public.js

@@ -1391,4 +1391,17 @@ actions.removePageData = (id: PageDataRemove) => {
   }
 }
 
+/**
+ * Record that a page was visited in the Client..
+ *
+ * @param {Object} $0
+ * @param {string} $0.id the chunkName for the page component.
+ */
+actions.createClientVisitedPage = (chunkName: string) => {
+  return {
+    type: `CREATE_CLIENT_VISITED_PAGE`,
+    payload: { componentChunkName: chunkName },
+  }
+}
+
 module.exports = { actions }

packages/gatsby/src/redux/reducers/client-visited-page.ts

@@ -0,0 +1,29 @@
+import {
+  IGatsbyState,
+  IDeleteCacheAction,
+  ICreateClientVisitedPage,
+} from "../types"
+
+// The develop server always wants these page components.
+const defaults = new Set<string>()
+defaults.add(`component---cache-dev-404-page-js`)
+defaults.add(`component---src-pages-404-js`)
+
+export const clientVisitedPageReducer = (
+  state: IGatsbyState["clientVisitedPages"] = new Set<string>(defaults),
+  action: IDeleteCacheAction | ICreateClientVisitedPage
+): IGatsbyState["clientVisitedPages"] => {
+  switch (action.type) {
+    case `DELETE_CACHE`:
+      return new Set<string>(defaults)
+
+    case `CREATE_CLIENT_VISITED_PAGE`: {
+      state.add(action.payload.componentChunkName)
+
+      return state
+    }
+
+    default:
+      return state
+  }
+}

packages/gatsby/src/redux/reducers/index.ts

@@ -27,6 +27,7 @@ import { schemaCustomizationReducer } from "./schema-customization"
 import { inferenceMetadataReducer } from "./inference-metadata"
 import { staticQueriesByTemplateReducer } from "./static-queries-by-template"
 import { queriesReducer } from "./queries"
+import { clientVisitedPageReducer } from "./client-visited-page"
 
 /**
  * @property exports.nodesTouched Set<string>
@@ -43,6 +44,7 @@ export {
   configReducer as config,
   schemaReducer as schema,
   pagesReducer as pages,
+  clientVisitedPageReducer as clientVisitedPages,
   statusReducer as status,
   componentsReducer as components,
   staticQueryComponentsReducer as staticQueryComponents,

packages/gatsby/src/redux/types.ts

@@ -274,6 +274,7 @@ export interface IGatsbyState {
   }
   pageDataStats: Map<SystemPath, number>
   pageData: Map<Identifier, string>
+  clientVisitedPages: Set<string>
 }
 
 export interface ICachedReduxState {
@@ -589,6 +590,12 @@ interface ISetSchemaComposerAction {
   payload: SchemaComposer<any>
 }
 
+export interface ICreateClientVisitedPage {
+  type: `CREATE_CLIENT_VISITED_PAGE`
+  payload: IGatsbyPage
+  plugin?: IGatsbyPlugin
+}
+
 export interface ICreatePageAction {
   type: `CREATE_PAGE`
   payload: IGatsbyPage