feat(gatsby): Defer node mutation during querying

packages/gatsby/src/commands/develop-process.ts

@@ -3,16 +3,10 @@ import reporter from "gatsby-cli/lib/reporter"
 import chalk from "chalk"
 import telemetry from "gatsby-telemetry"
 import express from "express"
-import { bootstrapSchemaHotReloader } from "../bootstrap/schema-hot-reloader"
-import bootstrapPageHotReloader from "../bootstrap/page-hot-reloader"
 import { initTracer } from "../utils/tracer"
 import db from "../db"
 import { detectPortInUseAndPrompt } from "../utils/detect-port-in-use-and-prompt"
 import onExit from "signal-exit"
-import queryUtil from "../query"
-import queryWatcher from "../query/query-watcher"
-import * as requiresWriter from "../bootstrap/requires-writer"
-import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete"
 import {
   userPassesFeedbackRequestHeuristic,
   showFeedbackRequest,
@@ -22,32 +16,33 @@ import { markWebpackStatusAsPending } from "../utils/webpack-status"
 
 import { IProgram } from "./types"
 import {
-  startWebpackServer,
-  writeOutRequires,
   IBuildContext,
   initialize,
-  postBootstrap,
   rebuildSchemaWithSitePage,
   writeOutRedirects,
+  startWebpackServer,
 } from "../services"
 import { boundActionCreators } from "../redux/actions"
 import { ProgramStatus } from "../redux/types"
 import {
   MachineConfig,
   AnyEventObject,
-  assign,
   Machine,
-  DoneEventObject,
   interpret,
   Actor,
   Interpreter,
+  forwardTo,
   State,
 } from "xstate"
-import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer"
+import { dataLayerMachine } from "../state-machines/data-layer"
 import { IDataLayerContext } from "../state-machines/data-layer/types"
 import { globalTracer } from "opentracing"
 import { IQueryRunningContext } from "../state-machines/query-running/types"
 import { queryRunningMachine } from "../state-machines/query-running"
+import { IWaitingContext } from "../state-machines/waiting/types"
+import { runMutationAndMarkDirty } from "../state-machines/shared-transition-configs"
+import { buildActions } from "../state-machines/actions"
+import { waitingMachine } from "../state-machines/waiting"
 
 const tracer = globalTracer()
 
@@ -85,6 +80,7 @@ process.on(`message`, msg => {
 })
 
 module.exports = async (program: IProgram): Promise<void> => {
+  reporter.setVerbose(program.verbose)
   const bootstrapSpan = tracer.startSpan(`bootstrap`)
 
   // We want to prompt the feedback request when users quit develop
@@ -135,31 +131,56 @@ module.exports = async (program: IProgram): Promise<void> => {
     initial: `initializing`,
     states: {
       initializing: {
+        on: {
+          // Ignore mutation events because we'll be running everything anyway
+          ADD_NODE_MUTATION: undefined,
+          QUERY_FILE_CHANGED: undefined,
+          WEBHOOK_RECEIVED: undefined,
+        },
         invoke: {
           src: `initialize`,
           onDone: {
             target: `initializingDataLayer`,
-            actions: `assignStoreAndWorkerPool`,
+            actions: [`assignStoreAndWorkerPool`, `spawnMutationListener`],
           },
         },
       },
       initializingDataLayer: {
+        on: {
+          ADD_NODE_MUTATION: runMutationAndMarkDirty,
+          // Ignore, because we're about to extract them anyway
+          QUERY_FILE_CHANGED: undefined,
+        },
         invoke: {
           src: `initializeDataLayer`,
-          data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => {
-            return { parentSpan, store, firstRun: true }
+          data: ({
+            parentSpan,
+            store,
+            firstRun,
+            webhookBody,
+          }: IBuildContext): IDataLayerContext => {
+            return {
+              parentSpan,
+              store,
+              firstRun,
+              deferNodeMutation: true,
+              webhookBody,
+            }
           },
           onDone: {
-            actions: `assignDataLayer`,
+            actions: [`assignServiceResult`, `clearWebhookBody`],
             target: `finishingBootstrap`,
           },
         },
       },
       finishingBootstrap: {
+        on: {
+          ADD_NODE_MUTATION: runMutationAndMarkDirty,
+          // Ignore, because we're about to extract them anyway
+          QUERY_FILE_CHANGED: undefined,
+        },
         invoke: {
-          src: async ({
-            gatsbyNodeGraphQLFunction,
-          }: IBuildContext): Promise<void> => {
+          src: async (): Promise<void> => {
             // These were previously in `bootstrap()` but are now
             // in part of the state machine that hasn't been added yet
             await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan })
@@ -168,30 +189,28 @@ module.exports = async (program: IProgram): Promise<void> => {
 
             startRedirectListener()
             bootstrapSpan.finish()
-            await postBootstrap({ parentSpan: bootstrapSpan })
-
-            // These are the parts that weren't in bootstrap
-
-            // Start the createPages hot reloader.
-            bootstrapPageHotReloader(gatsbyNodeGraphQLFunction)
-
-            // Start the schema hot reloader.
-            bootstrapSchemaHotReloader()
           },
           onDone: {
             target: `runningQueries`,
           },
         },
       },
       runningQueries: {
+        on: {
+          QUERY_FILE_CHANGED: {
+            actions: forwardTo(`run-queries`),
+          },
+        },
         invoke: {
+          id: `run-queries`,
           src: `runQueries`,
           data: ({
             program,
             store,
             parentSpan,
             gatsbyNodeGraphQLFunction,
             graphqlRunner,
+            websocketManager,
             firstRun,
           }: IBuildContext): IQueryRunningContext => {
             return {
@@ -201,6 +220,7 @@ module.exports = async (program: IProgram): Promise<void> => {
               parentSpan,
               gatsbyNodeGraphQLFunction,
               graphqlRunner,
+              websocketManager,
             }
           },
           onDone: {
@@ -210,57 +230,104 @@ module.exports = async (program: IProgram): Promise<void> => {
       },
       doingEverythingElse: {
         invoke: {
-          src: async ({ workerPool, store, app }): Promise<void> => {
+          src: async (): Promise<void> => {
             // All the stuff that's not in the state machine yet
 
-            await writeOutRequires({ store })
             boundActionCreators.setProgramStatus(
               ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED
             )
 
             await db.saveState()
 
-            await waitUntilAllJobsComplete()
-            requiresWriter.startListener()
             db.startAutosave()
-            queryUtil.startListeningToDevelopQueue({
-              graphqlTracing: program.graphqlTracing,
-            })
-            queryWatcher.startWatchDeletePage()
-
-            await startWebpackServer({ program, app, workerPool, store })
+          },
+          onDone: [
+            {
+              target: `startingDevServers`,
+              cond: ({ compiler }: IBuildContext): boolean => !compiler,
+            },
+            {
+              target: `waiting`,
+            },
+          ],
+        },
+      },
+      startingDevServers: {
+        invoke: {
+          src: `startWebpackServer`,
+          onDone: {
+            target: `waiting`,
+            actions: `assignServers`,
+          },
+        },
+      },
+      waiting: {
+        on: {
+          ADD_NODE_MUTATION: {
+            actions: forwardTo(`waiting`),
+          },
+          QUERY_FILE_CHANGED: {
+            actions: forwardTo(`waiting`),
+          },
+          EXTRACT_QUERIES_NOW: {
+            target: `runningQueries`,
+          },
+        },
+        invoke: {
+          id: `waiting`,
+          src: `waitForMutations`,
+          data: ({
+            store,
+            nodeMutationBatch = [],
+          }: IBuildContext): IWaitingContext => {
+            return { store, nodeMutationBatch, runningBatch: [] }
+          },
+          onDone: {
+            actions: `assignServiceResult`,
+            target: `rebuildingPages`,
+          },
+        },
+      },
+      rebuildingPages: {
+        invoke: {
+          src: `initializeDataLayer`,
+          data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => {
+            return { parentSpan, store, firstRun: false, skipSourcing: true }
           },
           onDone: {
-            actions: assign<IBuildContext, any>({ firstRun: false }),
+            actions: `assignServiceResult`,
+            target: `runningQueries`,
           },
         },
       },
     },
-  }
-
-  const service = interpret(
-    Machine(developConfig, {
-      services: {
-        initializeDataLayer: dataLayerMachine,
-        initialize,
-        runQueries: queryRunningMachine,
+    // Transitions shared by all states, except where overridden
+    on: {
+      ADD_NODE_MUTATION: {
+        actions: `addNodeMutation`,
       },
-      actions: {
-        assignStoreAndWorkerPool: assign<IBuildContext, DoneEventObject>(
-          (_context, event) => {
-            const { store, workerPool } = event.data
-            return {
-              store,
-              workerPool,
-            }
-          }
-        ),
-        assignDataLayer: assign<IBuildContext, DoneEventObject>(
-          (_, { data }): DataLayerResult => data
-        ),
+      QUERY_FILE_CHANGED: {
+        actions: `markQueryFilesDirty`,
       },
-    }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true })
-  )
+      WEBHOOK_RECEIVED: {
+        target: `initializingDataLayer`,
+        actions: `assignWebhookBody`,
+      },
+    },
+  }
+
+  const machine = Machine(developConfig, {
+    services: {
+      initializeDataLayer: dataLayerMachine,
+      initialize,
+      runQueries: queryRunningMachine,
+      waitForMutations: waitingMachine,
+      startWebpackServer: startWebpackServer,
+    },
+    actions: buildActions,
+  }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true })
+
+  const service = interpret(machine)
 
   const isInterpreter = <T>(
     actor: Actor<T> | Interpreter<T>

packages/gatsby/src/commands/types.ts

@@ -24,6 +24,7 @@ export interface IProgram {
   sitePackageJson: PackageJson
   ssl?: ICert
   graphqlTracing?: boolean
+  verbose?: boolean
   setStore?: (store: Store<IGatsbyState, AnyAction>) => void
 }
 

packages/gatsby/src/query/index.js

@@ -366,6 +366,7 @@ const enqueueExtractedPageComponent = componentPath => {
 
 module.exports = {
   calcInitialDirtyQueryIds,
+  calcDirtyQueryIds,
   processPageQueries,
   processStaticQueries,
   groupQueryIds,

packages/gatsby/src/query/query-watcher.js

@@ -8,7 +8,6 @@
  * - Whenever a query changes, re-run all pages that rely on this query.
  ***/
 
-const _ = require(`lodash`)
 const chokidar = require(`chokidar`)
 
 const path = require(`path`)
@@ -196,8 +195,7 @@ exports.extractQueries = ({ parentSpan } = {}) => {
     // During development start watching files to recompile & run
     // queries on the fly.
 
-    // TODO: move this into a spawned service, and emit events rather than
-    // directly triggering the compilation
+    // TODO: move this into a spawned service
     if (process.env.NODE_ENV !== `production`) {
       watch(store.getState().program.directory)
     }
@@ -222,10 +220,6 @@ const watchComponent = componentPath => {
   }
 }
 
-const debounceCompile = _.debounce(() => {
-  updateStateAndRunQueries()
-}, 100)
-
 const watch = async rootDir => {
   if (watcher) return
 
@@ -238,13 +232,18 @@ const watch = async rootDir => {
   })
 
   watcher = chokidar
-    .watch([
-      slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)),
-      ...packagePaths,
-    ])
+    .watch(
+      [slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)), ...packagePaths],
+      { ignoreInitial: true }
+    )
     .on(`change`, path => {
-      report.pendingActivity({ id: `query-extraction` })
-      debounceCompile()
+      emitter.emit(`QUERY_FILE_CHANGED`, path)
+    })
+    .on(`add`, path => {
+      emitter.emit(`QUERY_FILE_CHANGED`, path)
+    })
+    .on(`unlink`, path => {
+      emitter.emit(`QUERY_FILE_CHANGED`, path)
     })
 
   filesToWatch.forEach(filePath => watcher.add(filePath))