feat(gatsby): Use state machine for query running in develop (#25378)

* Move bootstrap into machine

* Add parent span and query extraction

* Add rebuildSchemaWithSitePage

* Use values from context

* Remove logs

* Add redirectListener

* Changes from review

* Log child state transitions

* Add state machine for query running

* Changes from review

* Changes from review

* Switch to reporter

* Use assertStore

* Remove unused action

* Remove unusued config

* Remove unusued config

* Add gql runner reset

* Use new pagedata utils

* Use develop queue

* Track first run

* Changes from review

packages/gatsby/src/bootstrap/index.ts

@@ -19,7 +19,7 @@ import JestWorker from "jest-worker"
 const tracer = globalTracer()
 
 export async function bootstrap(
-  initialContext: IBuildContext
+  initialContext: Partial<IBuildContext>
 ): Promise<{
   gatsbyNodeGraphQLFunction: Runner
   workerPool: JestWorker
@@ -28,11 +28,17 @@ export async function bootstrap(
     ? { childOf: initialContext.parentSpan }
     : {}
 
-  initialContext.parentSpan = tracer.startSpan(`bootstrap`, spanArgs)
+  const parentSpan = tracer.startSpan(`bootstrap`, spanArgs)
 
-  const context = {
+  const bootstrapContext: IBuildContext = {
     ...initialContext,
-    ...(await initialize(initialContext)),
+    parentSpan,
+    firstRun: true,
+  }
+
+  const context = {
+    ...bootstrapContext,
+    ...(await initialize(bootstrapContext)),
   }
 
   await customizeSchema(context)
@@ -59,7 +65,7 @@ export async function bootstrap(
 
   await postBootstrap(context)
 
-  initialContext.parentSpan.finish()
+  parentSpan.finish()
 
   return {
     gatsbyNodeGraphQLFunction: context.gatsbyNodeGraphQLFunction,

packages/gatsby/src/commands/develop-process.ts

@@ -1,5 +1,5 @@
 import { syncStaticDir } from "../utils/get-static-dir"
-import report from "gatsby-cli/lib/reporter"
+import reporter from "gatsby-cli/lib/reporter"
 import chalk from "chalk"
 import telemetry from "gatsby-telemetry"
 import express from "express"
@@ -22,15 +22,11 @@ import { markWebpackStatusAsPending } from "../utils/webpack-status"
 
 import { IProgram } from "./types"
 import {
-  calculateDirtyQueries,
-  runStaticQueries,
-  runPageQueries,
   startWebpackServer,
   writeOutRequires,
   IBuildContext,
   initialize,
   postBootstrap,
-  extractQueries,
   rebuildSchemaWithSitePage,
   writeOutRedirects,
 } from "../services"
@@ -43,11 +39,15 @@ import {
   Machine,
   DoneEventObject,
   interpret,
+  Actor,
+  Interpreter,
+  State,
 } from "xstate"
 import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer"
 import { IDataLayerContext } from "../state-machines/data-layer/types"
 import { globalTracer } from "opentracing"
-import reporter from "gatsby-cli/lib/reporter"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
+import { queryRunningMachine } from "../state-machines/query-running"
 
 const tracer = globalTracer()
 
@@ -101,7 +101,7 @@ module.exports = async (program: IProgram): Promise<void> => {
   )
 
   if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
-    report.panic(
+    reporter.panic(
       `The flag ${chalk.yellow(
         `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES`
       )} is not available with ${chalk.cyan(
@@ -111,7 +111,7 @@ module.exports = async (program: IProgram): Promise<void> => {
   }
   initTracer(program.openTracingConfigFile)
   markWebpackStatusAsPending()
-  report.pendingActivity({ id: `webpack-develop` })
+  reporter.pendingActivity({ id: `webpack-develop` })
   telemetry.trackCli(`DEVELOP_START`)
   telemetry.startBackgroundUpdate()
 
@@ -151,26 +151,19 @@ module.exports = async (program: IProgram): Promise<void> => {
           },
           onDone: {
             actions: `assignDataLayer`,
-            target: `doingEverythingElse`,
+            target: `finishingBootstrap`,
           },
         },
       },
-      doingEverythingElse: {
+      finishingBootstrap: {
         invoke: {
           src: async ({
             gatsbyNodeGraphQLFunction,
-            graphqlRunner,
-            workerPool,
-            store,
-            app,
-          }): Promise<void> => {
-            // All the stuff that's not in the state machine yet
-
+          }: IBuildContext): Promise<void> => {
             // These were previously in `bootstrap()` but are now
             // in part of the state machine that hasn't been added yet
             await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan })
 
-            await extractQueries({ parentSpan: bootstrapSpan })
             await writeOutRedirects({ parentSpan: bootstrapSpan })
 
             startRedirectListener()
@@ -184,11 +177,42 @@ module.exports = async (program: IProgram): Promise<void> => {
 
             // Start the schema hot reloader.
             bootstrapSchemaHotReloader()
+          },
+          onDone: {
+            target: `runningQueries`,
+          },
+        },
+      },
+      runningQueries: {
+        invoke: {
+          src: `runQueries`,
+          data: ({
+            program,
+            store,
+            parentSpan,
+            gatsbyNodeGraphQLFunction,
+            graphqlRunner,
+            firstRun,
+          }: IBuildContext): IQueryRunningContext => {
+            return {
+              firstRun,
+              program,
+              store,
+              parentSpan,
+              gatsbyNodeGraphQLFunction,
+              graphqlRunner,
+            }
+          },
+          onDone: {
+            target: `doingEverythingElse`,
+          },
+        },
+      },
+      doingEverythingElse: {
+        invoke: {
+          src: async ({ workerPool, store, app }): Promise<void> => {
+            // All the stuff that's not in the state machine yet
 
-            const { queryIds } = await calculateDirtyQueries({ store })
-
-            await runStaticQueries({ queryIds, store, program, graphqlRunner })
-            await runPageQueries({ queryIds, store, program, graphqlRunner })
             await writeOutRequires({ store })
             boundActionCreators.setProgramStatus(
               ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED
@@ -206,17 +230,20 @@ module.exports = async (program: IProgram): Promise<void> => {
 
             await startWebpackServer({ program, app, workerPool })
           },
+          onDone: {
+            actions: assign<IBuildContext, any>({ firstRun: false }),
+          },
         },
       },
     },
   }
 
   const service = interpret(
-    // eslint-disable-next-line new-cap
     Machine(developConfig, {
       services: {
         initializeDataLayer: dataLayerMachine,
         initialize,
+        runQueries: queryRunningMachine,
       },
       actions: {
         assignStoreAndWorkerPool: assign<IBuildContext, DoneEventObject>(
@@ -232,10 +259,48 @@ module.exports = async (program: IProgram): Promise<void> => {
           (_, { data }): DataLayerResult => data
         ),
       },
-    }).withContext({ program, parentSpan: bootstrapSpan, app })
+    }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true })
   )
+
+  const isInterpreter = <T>(
+    actor: Actor<T> | Interpreter<T>
+  ): actor is Interpreter<T> => `machine` in actor
+
+  const listeners = new WeakSet()
+  let last: State<IBuildContext, AnyEventObject, any, any>
+
   service.onTransition(state => {
-    reporter.verbose(`transition to ${JSON.stringify(state.value)}`)
+    if (!last) {
+      last = state
+    } else if (!state.changed || last.matches(state)) {
+      return
+    }
+    last = state
+    reporter.verbose(`Transition to ${JSON.stringify(state.value)}`)
+    // eslint-disable-next-line no-unused-expressions
+    service.children?.forEach(child => {
+      // We want to ensure we don't attach a listener to the same
+      // actor. We don't need to worry about detaching the listener
+      // because xstate handles that for us when the actor is stopped.
+
+      if (isInterpreter(child) && !listeners.has(child)) {
+        let sublast = child.state
+        child.onTransition(substate => {
+          if (!sublast) {
+            sublast = substate
+          } else if (!substate.changed || sublast.matches(substate)) {
+            return
+          }
+          sublast = substate
+          reporter.verbose(
+            `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify(
+              substate.value
+            )}`
+          )
+        })
+        listeners.add(child)
+      }
+    })
   })
   service.start()
 }

packages/gatsby/src/query/index.js

@@ -166,7 +166,9 @@ const processQueries = async (
   queryJobs,
   { activity, graphqlRunner, graphqlTracing }
 ) => {
-  const queue = queryQueue.createBuildQueue(graphqlRunner, { graphqlTracing })
+  const queue = queryQueue.createAppropriateQueue(graphqlRunner, {
+    graphqlTracing,
+  })
   return queryQueue.processBatch(queue, queryJobs, activity)
 }
 

packages/gatsby/src/query/query-watcher.js

@@ -195,6 +195,9 @@ exports.extractQueries = ({ parentSpan } = {}) => {
   return updateStateAndRunQueries(true, { parentSpan }).then(() => {
     // During development start watching files to recompile & run
     // queries on the fly.
+
+    // TODO: move this into a spawned service, and emit events rather than
+    // directly triggering the compilation
     if (process.env.NODE_ENV !== `production`) {
       watch(store.getState().program.directory)
     }
@@ -252,7 +255,7 @@ exports.startWatchDeletePage = () => {
     const componentPath = slash(action.payload.component)
     const { pages } = store.getState()
     let otherPageWithTemplateExists = false
-    for (let page of pages.values()) {
+    for (const page of pages.values()) {
       if (slash(page.component) === componentPath) {
         otherPageWithTemplateExists = true
         break

packages/gatsby/src/query/queue.js

@@ -59,6 +59,16 @@ const createDevelopQueue = getRunner => {
   return new Queue(handler, queueOptions)
 }
 
+const createAppropriateQueue = (graphqlRunner, runnerOptions = {}) => {
+  if (process.env.NODE_ENV === `production`) {
+    return createBuildQueue(graphqlRunner, runnerOptions)
+  }
+  if (!graphqlRunner) {
+    graphqlRunner = new GraphQLRunner(store, runnerOptions)
+  }
+  return createDevelopQueue(() => graphqlRunner)
+}
+
 /**
  * Returns a promise that pushes jobs onto queue and resolves onces
  * they're all finished processing (or rejects if one or more jobs
@@ -116,4 +126,5 @@ module.exports = {
   createBuildQueue,
   createDevelopQueue,
   processBatch,
+  createAppropriateQueue,
 }

packages/gatsby/src/services/calculate-dirty-queries.ts

@@ -1,10 +1,11 @@
 import { calcInitialDirtyQueryIds, groupQueryIds } from "../query"
-import { IBuildContext, IGroupedQueryIds } from "./"
+import { IGroupedQueryIds } from "./"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
 import { assertStore } from "../utils/assert-store"
 
 export async function calculateDirtyQueries({
   store,
-}: Partial<IBuildContext>): Promise<{
+}: Partial<IQueryRunningContext>): Promise<{
   queryIds: IGroupedQueryIds
 }> {
   assertStore(store)

packages/gatsby/src/services/extract-queries.ts

@@ -1,12 +1,11 @@
-import { IBuildContext } from "./"
-
 import reporter from "gatsby-cli/lib/reporter"
 import { extractQueries as extractQueriesAndWatch } from "../query/query-watcher"
 import apiRunnerNode from "../utils/api-runner-node"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
 
 export async function extractQueries({
   parentSpan,
-}: Partial<IBuildContext>): Promise<void> {
+}: Partial<IQueryRunningContext>): Promise<void> {
   const activity = reporter.activityTimer(`onPreExtractQueries`, {
     parentSpan,
   })

packages/gatsby/src/services/rebuild-schema-with-site-pages.ts

@@ -1,10 +1,10 @@
-import { IBuildContext } from "./"
 import { rebuildWithSitePage } from "../schema"
 import reporter from "gatsby-cli/lib/reporter"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
 
 export async function rebuildSchemaWithSitePage({
   parentSpan,
-}: Partial<IBuildContext>): Promise<void> {
+}: Partial<IQueryRunningContext>): Promise<void> {
   const activity = reporter.activityTimer(`updating schema`, {
     parentSpan,
   })

packages/gatsby/src/services/run-page-queries.ts

@@ -1,6 +1,6 @@
 import { processPageQueries } from "../query"
-import { IBuildContext } from "./"
 import reporter from "gatsby-cli/lib/reporter"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
 import { assertStore } from "../utils/assert-store"
 
 export async function runPageQueries({
@@ -9,7 +9,7 @@ export async function runPageQueries({
   store,
   program,
   graphqlRunner,
-}: Partial<IBuildContext>): Promise<void> {
+}: Partial<IQueryRunningContext>): Promise<void> {
   assertStore(store)
 
   if (!queryIds) {

packages/gatsby/src/services/run-static-queries.ts

@@ -1,6 +1,6 @@
 import { processStaticQueries } from "../query"
-import { IBuildContext } from "./"
 import reporter from "gatsby-cli/lib/reporter"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
 import { assertStore } from "../utils/assert-store"
 
 export async function runStaticQueries({
@@ -9,7 +9,7 @@ export async function runStaticQueries({
   store,
   program,
   graphqlRunner,
-}: Partial<IBuildContext>): Promise<void> {
+}: Partial<IQueryRunningContext>): Promise<void> {
   assertStore(store)
 
   if (!queryIds) {

packages/gatsby/src/services/types.ts

@@ -16,6 +16,7 @@ export interface IMutationAction {
   payload: unknown[]
 }
 export interface IBuildContext {
+  firstRun: boolean
   program?: IProgram
   store?: Store<IGatsbyState, AnyAction>
   parentSpan?: Span

packages/gatsby/src/services/write-out-redirects.ts

@@ -1,10 +1,10 @@
 import reporter from "gatsby-cli/lib/reporter"
 import { writeRedirects } from "../bootstrap/redirects-writer"
-import { IDataLayerContext } from "../state-machines/data-layer/types"
+import { IQueryRunningContext } from "../state-machines/query-running/types"
 
 export async function writeOutRedirects({
   parentSpan,
-}: Partial<IDataLayerContext>): Promise<void> {
+}: Partial<IQueryRunningContext>): Promise<void> {
   // Write out redirects.
   const activity = reporter.activityTimer(`write out redirect data`, {
     parentSpan,