From e5ce35b9456784288634f2070a0f319b8fe76873 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 7 Jul 2020 10:06:50 +0100 Subject: [PATCH] feat(gatsby): Use state machine for query running in develop (#25378) * Move bootstrap into machine * Add parent span and query extraction * Add rebuildSchemaWithSitePage * Use values from context * Remove logs * Add redirectListener * Changes from review * Log child state transitions * Add state machine for query running * Changes from review * Changes from review * Switch to reporter * Use assertStore * Remove unused action * Remove unusued config * Remove unusued config * Add gql runner reset * Use new pagedata utils * Use develop queue * Track first run * Changes from review --- packages/gatsby/src/bootstrap/index.ts | 16 ++- .../gatsby/src/commands/develop-process.ts | 115 ++++++++++++++---- packages/gatsby/src/query/index.js | 4 +- packages/gatsby/src/query/query-watcher.js | 5 +- packages/gatsby/src/query/queue.js | 11 ++ .../src/services/calculate-dirty-queries.ts | 5 +- .../gatsby/src/services/extract-queries.ts | 5 +- .../rebuild-schema-with-site-pages.ts | 4 +- .../gatsby/src/services/run-page-queries.ts | 4 +- .../gatsby/src/services/run-static-queries.ts | 4 +- packages/gatsby/src/services/types.ts | 1 + .../src/services/write-out-redirects.ts | 4 +- .../gatsby/src/services/write-out-requires.ts | 4 +- .../state-machines/query-running/actions.ts | 42 +++++++ .../src/state-machines/query-running/index.ts | 78 ++++++++++++ .../state-machines/query-running/services.ts | 26 ++++ .../src/state-machines/query-running/types.ts | 22 ++++ 17 files changed, 303 insertions(+), 47 deletions(-) create mode 100644 packages/gatsby/src/state-machines/query-running/actions.ts create mode 100644 packages/gatsby/src/state-machines/query-running/index.ts create mode 100644 packages/gatsby/src/state-machines/query-running/services.ts create mode 100644 packages/gatsby/src/state-machines/query-running/types.ts diff --git a/packages/gatsby/src/bootstrap/index.ts b/packages/gatsby/src/bootstrap/index.ts index 7cb9632a23c21..0bb37c8b7ab9c 100644 --- a/packages/gatsby/src/bootstrap/index.ts +++ b/packages/gatsby/src/bootstrap/index.ts @@ -19,7 +19,7 @@ import JestWorker from "jest-worker" const tracer = globalTracer() export async function bootstrap( - initialContext: IBuildContext + initialContext: Partial ): Promise<{ gatsbyNodeGraphQLFunction: Runner workerPool: JestWorker @@ -28,11 +28,17 @@ export async function bootstrap( ? { childOf: initialContext.parentSpan } : {} - initialContext.parentSpan = tracer.startSpan(`bootstrap`, spanArgs) + const parentSpan = tracer.startSpan(`bootstrap`, spanArgs) - const context = { + const bootstrapContext: IBuildContext = { ...initialContext, - ...(await initialize(initialContext)), + parentSpan, + firstRun: true, + } + + const context = { + ...bootstrapContext, + ...(await initialize(bootstrapContext)), } await customizeSchema(context) @@ -59,7 +65,7 @@ export async function bootstrap( await postBootstrap(context) - initialContext.parentSpan.finish() + parentSpan.finish() return { gatsbyNodeGraphQLFunction: context.gatsbyNodeGraphQLFunction, diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 7305660a51994..d7e6212c23bc6 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -1,5 +1,5 @@ import { syncStaticDir } from "../utils/get-static-dir" -import report from "gatsby-cli/lib/reporter" +import reporter from "gatsby-cli/lib/reporter" import chalk from "chalk" import telemetry from "gatsby-telemetry" import express from "express" @@ -22,15 +22,11 @@ import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram } from "./types" import { - calculateDirtyQueries, - runStaticQueries, - runPageQueries, startWebpackServer, writeOutRequires, IBuildContext, initialize, postBootstrap, - extractQueries, rebuildSchemaWithSitePage, writeOutRedirects, } from "../services" @@ -43,11 +39,15 @@ import { Machine, DoneEventObject, interpret, + Actor, + Interpreter, + State, } from "xstate" import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" import { globalTracer } from "opentracing" -import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" +import { queryRunningMachine } from "../state-machines/query-running" const tracer = globalTracer() @@ -101,7 +101,7 @@ module.exports = async (program: IProgram): Promise => { ) if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) { - report.panic( + reporter.panic( `The flag ${chalk.yellow( `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES` )} is not available with ${chalk.cyan( @@ -111,7 +111,7 @@ module.exports = async (program: IProgram): Promise => { } initTracer(program.openTracingConfigFile) markWebpackStatusAsPending() - report.pendingActivity({ id: `webpack-develop` }) + reporter.pendingActivity({ id: `webpack-develop` }) telemetry.trackCli(`DEVELOP_START`) telemetry.startBackgroundUpdate() @@ -151,26 +151,19 @@ module.exports = async (program: IProgram): Promise => { }, onDone: { actions: `assignDataLayer`, - target: `doingEverythingElse`, + target: `finishingBootstrap`, }, }, }, - doingEverythingElse: { + finishingBootstrap: { invoke: { src: async ({ gatsbyNodeGraphQLFunction, - graphqlRunner, - workerPool, - store, - app, - }): Promise => { - // All the stuff that's not in the state machine yet - + }: IBuildContext): Promise => { // These were previously in `bootstrap()` but are now // in part of the state machine that hasn't been added yet await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) - await extractQueries({ parentSpan: bootstrapSpan }) await writeOutRedirects({ parentSpan: bootstrapSpan }) startRedirectListener() @@ -184,11 +177,42 @@ module.exports = async (program: IProgram): Promise => { // Start the schema hot reloader. bootstrapSchemaHotReloader() + }, + onDone: { + target: `runningQueries`, + }, + }, + }, + runningQueries: { + invoke: { + src: `runQueries`, + data: ({ + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + firstRun, + }: IBuildContext): IQueryRunningContext => { + return { + firstRun, + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + } + }, + onDone: { + target: `doingEverythingElse`, + }, + }, + }, + doingEverythingElse: { + invoke: { + src: async ({ workerPool, store, app }): Promise => { + // All the stuff that's not in the state machine yet - const { queryIds } = await calculateDirtyQueries({ store }) - - await runStaticQueries({ queryIds, store, program, graphqlRunner }) - await runPageQueries({ queryIds, store, program, graphqlRunner }) await writeOutRequires({ store }) boundActionCreators.setProgramStatus( ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED @@ -206,17 +230,20 @@ module.exports = async (program: IProgram): Promise => { await startWebpackServer({ program, app, workerPool }) }, + onDone: { + actions: assign({ firstRun: false }), + }, }, }, }, } const service = interpret( - // eslint-disable-next-line new-cap Machine(developConfig, { services: { initializeDataLayer: dataLayerMachine, initialize, + runQueries: queryRunningMachine, }, actions: { assignStoreAndWorkerPool: assign( @@ -232,10 +259,48 @@ module.exports = async (program: IProgram): Promise => { (_, { data }): DataLayerResult => data ), }, - }).withContext({ program, parentSpan: bootstrapSpan, app }) + }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) ) + + const isInterpreter = ( + actor: Actor | Interpreter + ): actor is Interpreter => `machine` in actor + + const listeners = new WeakSet() + let last: State + service.onTransition(state => { - reporter.verbose(`transition to ${JSON.stringify(state.value)}`) + if (!last) { + last = state + } else if (!state.changed || last.matches(state)) { + return + } + last = state + reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) + // eslint-disable-next-line no-unused-expressions + service.children?.forEach(child => { + // We want to ensure we don't attach a listener to the same + // actor. We don't need to worry about detaching the listener + // because xstate handles that for us when the actor is stopped. + + if (isInterpreter(child) && !listeners.has(child)) { + let sublast = child.state + child.onTransition(substate => { + if (!sublast) { + sublast = substate + } else if (!substate.changed || sublast.matches(substate)) { + return + } + sublast = substate + reporter.verbose( + `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( + substate.value + )}` + ) + }) + listeners.add(child) + } + }) }) service.start() } diff --git a/packages/gatsby/src/query/index.js b/packages/gatsby/src/query/index.js index 88de7ab91b880..a8c9d7a96773b 100644 --- a/packages/gatsby/src/query/index.js +++ b/packages/gatsby/src/query/index.js @@ -166,7 +166,9 @@ const processQueries = async ( queryJobs, { activity, graphqlRunner, graphqlTracing } ) => { - const queue = queryQueue.createBuildQueue(graphqlRunner, { graphqlTracing }) + const queue = queryQueue.createAppropriateQueue(graphqlRunner, { + graphqlTracing, + }) return queryQueue.processBatch(queue, queryJobs, activity) } diff --git a/packages/gatsby/src/query/query-watcher.js b/packages/gatsby/src/query/query-watcher.js index cc16b7c1d4f2f..d20676a3feea3 100644 --- a/packages/gatsby/src/query/query-watcher.js +++ b/packages/gatsby/src/query/query-watcher.js @@ -195,6 +195,9 @@ exports.extractQueries = ({ parentSpan } = {}) => { return updateStateAndRunQueries(true, { parentSpan }).then(() => { // During development start watching files to recompile & run // queries on the fly. + + // TODO: move this into a spawned service, and emit events rather than + // directly triggering the compilation if (process.env.NODE_ENV !== `production`) { watch(store.getState().program.directory) } @@ -252,7 +255,7 @@ exports.startWatchDeletePage = () => { const componentPath = slash(action.payload.component) const { pages } = store.getState() let otherPageWithTemplateExists = false - for (let page of pages.values()) { + for (const page of pages.values()) { if (slash(page.component) === componentPath) { otherPageWithTemplateExists = true break diff --git a/packages/gatsby/src/query/queue.js b/packages/gatsby/src/query/queue.js index ed80fb7d6cbf6..104a19e05c764 100644 --- a/packages/gatsby/src/query/queue.js +++ b/packages/gatsby/src/query/queue.js @@ -59,6 +59,16 @@ const createDevelopQueue = getRunner => { return new Queue(handler, queueOptions) } +const createAppropriateQueue = (graphqlRunner, runnerOptions = {}) => { + if (process.env.NODE_ENV === `production`) { + return createBuildQueue(graphqlRunner, runnerOptions) + } + if (!graphqlRunner) { + graphqlRunner = new GraphQLRunner(store, runnerOptions) + } + return createDevelopQueue(() => graphqlRunner) +} + /** * Returns a promise that pushes jobs onto queue and resolves onces * they're all finished processing (or rejects if one or more jobs @@ -116,4 +126,5 @@ module.exports = { createBuildQueue, createDevelopQueue, processBatch, + createAppropriateQueue, } diff --git a/packages/gatsby/src/services/calculate-dirty-queries.ts b/packages/gatsby/src/services/calculate-dirty-queries.ts index cc5802067110a..adf3bd7501bb5 100644 --- a/packages/gatsby/src/services/calculate-dirty-queries.ts +++ b/packages/gatsby/src/services/calculate-dirty-queries.ts @@ -1,10 +1,11 @@ import { calcInitialDirtyQueryIds, groupQueryIds } from "../query" -import { IBuildContext, IGroupedQueryIds } from "./" +import { IGroupedQueryIds } from "./" +import { IQueryRunningContext } from "../state-machines/query-running/types" import { assertStore } from "../utils/assert-store" export async function calculateDirtyQueries({ store, -}: Partial): Promise<{ +}: Partial): Promise<{ queryIds: IGroupedQueryIds }> { assertStore(store) diff --git a/packages/gatsby/src/services/extract-queries.ts b/packages/gatsby/src/services/extract-queries.ts index 30441f904a084..425cd506e402c 100644 --- a/packages/gatsby/src/services/extract-queries.ts +++ b/packages/gatsby/src/services/extract-queries.ts @@ -1,12 +1,11 @@ -import { IBuildContext } from "./" - import reporter from "gatsby-cli/lib/reporter" import { extractQueries as extractQueriesAndWatch } from "../query/query-watcher" import apiRunnerNode from "../utils/api-runner-node" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function extractQueries({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { const activity = reporter.activityTimer(`onPreExtractQueries`, { parentSpan, }) diff --git a/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts b/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts index a58c5fdb76410..c5b41c4859886 100644 --- a/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts +++ b/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts @@ -1,10 +1,10 @@ -import { IBuildContext } from "./" import { rebuildWithSitePage } from "../schema" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function rebuildSchemaWithSitePage({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { const activity = reporter.activityTimer(`updating schema`, { parentSpan, }) diff --git a/packages/gatsby/src/services/run-page-queries.ts b/packages/gatsby/src/services/run-page-queries.ts index 90789465fdfb1..528eb83949377 100644 --- a/packages/gatsby/src/services/run-page-queries.ts +++ b/packages/gatsby/src/services/run-page-queries.ts @@ -1,6 +1,6 @@ import { processPageQueries } from "../query" -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" import { assertStore } from "../utils/assert-store" export async function runPageQueries({ @@ -9,7 +9,7 @@ export async function runPageQueries({ store, program, graphqlRunner, -}: Partial): Promise { +}: Partial): Promise { assertStore(store) if (!queryIds) { diff --git a/packages/gatsby/src/services/run-static-queries.ts b/packages/gatsby/src/services/run-static-queries.ts index f11882f05eb3d..4dbaaedf8c5e8 100644 --- a/packages/gatsby/src/services/run-static-queries.ts +++ b/packages/gatsby/src/services/run-static-queries.ts @@ -1,6 +1,6 @@ import { processStaticQueries } from "../query" -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" import { assertStore } from "../utils/assert-store" export async function runStaticQueries({ @@ -9,7 +9,7 @@ export async function runStaticQueries({ store, program, graphqlRunner, -}: Partial): Promise { +}: Partial): Promise { assertStore(store) if (!queryIds) { diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 3ec2a3766f28a..1b961b5479370 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -16,6 +16,7 @@ export interface IMutationAction { payload: unknown[] } export interface IBuildContext { + firstRun: boolean program?: IProgram store?: Store parentSpan?: Span diff --git a/packages/gatsby/src/services/write-out-redirects.ts b/packages/gatsby/src/services/write-out-redirects.ts index d7c30f690e390..5745db6dc6fdf 100644 --- a/packages/gatsby/src/services/write-out-redirects.ts +++ b/packages/gatsby/src/services/write-out-redirects.ts @@ -1,10 +1,10 @@ import reporter from "gatsby-cli/lib/reporter" import { writeRedirects } from "../bootstrap/redirects-writer" -import { IDataLayerContext } from "../state-machines/data-layer/types" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function writeOutRedirects({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { // Write out redirects. const activity = reporter.activityTimer(`write out redirect data`, { parentSpan, diff --git a/packages/gatsby/src/services/write-out-requires.ts b/packages/gatsby/src/services/write-out-requires.ts index f016f8e3f46ac..0b20d598c1203 100644 --- a/packages/gatsby/src/services/write-out-requires.ts +++ b/packages/gatsby/src/services/write-out-requires.ts @@ -1,12 +1,12 @@ -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" import { writeAll } from "../bootstrap/requires-writer" +import { IQueryRunningContext } from "../state-machines/query-running/types" import { assertStore } from "../utils/assert-store" export async function writeOutRequires({ store, parentSpan, -}: Partial): Promise { +}: Partial): Promise { assertStore(store) // Write out files. diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts new file mode 100644 index 0000000000000..c329e95ea8356 --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -0,0 +1,42 @@ +import { IQueryRunningContext } from "./types" +import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" +import { GraphQLRunner } from "../../query/graphql-runner" +import { assertStore } from "../../utils/assert-store" +import { enqueueFlush } from "../../utils/page-data" + +export const flushPageData = (): void => { + enqueueFlush() +} + +export const assignDirtyQueries = assign< + IQueryRunningContext, + DoneInvokeEvent +>((_context, { data }) => { + const { queryIds } = data + return { + filesDirty: false, + queryIds, + } +}) + +export const resetGraphQLRunner = assign< + IQueryRunningContext, + DoneInvokeEvent +>({ + graphqlRunner: ({ store, program }) => { + assertStore(store) + return new GraphQLRunner(store, { + collectStats: true, + graphqlTracing: program?.graphqlTracing, + }) + }, +}) + +export const queryActions: ActionFunctionMap< + IQueryRunningContext, + DoneInvokeEvent +> = { + resetGraphQLRunner, + assignDirtyQueries, + flushPageData, +} diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts new file mode 100644 index 0000000000000..39161dac73304 --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -0,0 +1,78 @@ +import { MachineConfig, Machine } from "xstate" +import { IQueryRunningContext } from "./types" +import { queryRunningServices } from "./services" +import { queryActions } from "./actions" + +export const queryStates: MachineConfig = { + initial: `extractingQueries`, + states: { + extractingQueries: { + id: `extracting-queries`, + invoke: { + id: `extracting-queries`, + src: `extractQueries`, + onDone: [ + { + actions: `resetGraphQLRunner`, + target: `writingRequires`, + }, + ], + }, + }, + writingRequires: { + invoke: { + src: `writeOutRequires`, + id: `writing-requires`, + onDone: { + target: `calculatingDirtyQueries`, + }, + }, + }, + calculatingDirtyQueries: { + invoke: { + id: `calculating-dirty-queries`, + src: `calculateDirtyQueries`, + onDone: { + target: `runningStaticQueries`, + actions: `assignDirtyQueries`, + }, + }, + }, + runningStaticQueries: { + invoke: { + src: `runStaticQueries`, + id: `running-static-queries`, + onDone: { + target: `runningPageQueries`, + }, + }, + }, + runningPageQueries: { + invoke: { + src: `runPageQueries`, + id: `running-page-queries`, + onDone: { + target: `waitingForJobs`, + actions: `flushPageData`, + }, + }, + }, + + waitingForJobs: { + invoke: { + src: `waitUntilAllJobsComplete`, + id: `waiting-for-jobs`, + onDone: { + target: `done`, + }, + }, + }, + done: { + type: `final`, + }, + }, +} +export const queryRunningMachine = Machine(queryStates, { + actions: queryActions, + services: queryRunningServices, +}) diff --git a/packages/gatsby/src/state-machines/query-running/services.ts b/packages/gatsby/src/state-machines/query-running/services.ts new file mode 100644 index 0000000000000..e5b7aaa1b2794 --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/services.ts @@ -0,0 +1,26 @@ +import { ServiceConfig } from "xstate" +import { + extractQueries, + writeOutRequires, + calculateDirtyQueries, + runStaticQueries, + runPageQueries, + waitUntilAllJobsComplete, + writeOutRedirects, + rebuildSchemaWithSitePage, +} from "../../services" +import { IQueryRunningContext } from "./types" + +export const queryRunningServices: Record< + string, + ServiceConfig +> = { + extractQueries, + writeOutRequires, + calculateDirtyQueries, + runStaticQueries, + runPageQueries, + waitUntilAllJobsComplete, + writeOutRedirects, + rebuildSchemaWithSitePage, +} diff --git a/packages/gatsby/src/state-machines/query-running/types.ts b/packages/gatsby/src/state-machines/query-running/types.ts new file mode 100644 index 0000000000000..d977fd967111b --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/types.ts @@ -0,0 +1,22 @@ +import { Span } from "opentracing" +import { IProgram } from "../../commands/types" +import { Runner } from "../../bootstrap/create-graphql-runner" +import { GraphQLRunner } from "../../query/graphql-runner" +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../../redux/types" +import { IGroupedQueryIds } from "../data-layer/types" +import { WebsocketManager } from "../../utils/websocket-manager" + +export interface IQueryRunningContext { + firstRun?: boolean + program?: IProgram + store?: Store + parentSpan?: Span + gatsbyNodeGraphQLFunction?: Runner + graphqlRunner?: GraphQLRunner + pagesToBuild?: string[] + pagesToDelete?: string[] + queryIds?: IGroupedQueryIds + websocketManager?: WebsocketManager + filesDirty?: boolean +}