From a5cd72a5adf4ce13ece5d14df1f3ae2f696fa8e3 Mon Sep 17 00:00:00 2001 From: Josh Johnson Date: Fri, 11 Mar 2022 06:43:31 -0500 Subject: [PATCH] fix(gatsby): use lmdb for resultHash cache so shared across workers (#34925) * use lmdb for resultHash cache so shared across workers * Clear lmdb before workers start processes queries * Pull db name into const * Undo reset changes, as it's not actually needed * Undo accidental e2e change * Add to pagepaths if component path/id was modified * Update snapshots * Move lmdb creation into function * require -> import Co-authored-by: Kyle Mathews Co-authored-by: Michal Piechowiak --- .../src/query/__tests__/data-tracking.js | 1 + packages/gatsby/src/query/query-runner.ts | 19 +++++++++++++++---- .../__tests__/__snapshots__/pages.ts.snap | 3 +++ packages/gatsby/src/redux/actions/public.js | 3 +++ .../reducers/pending-page-data-writes.ts | 7 +++++++ packages/gatsby/src/redux/types.ts | 1 + 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/gatsby/src/query/__tests__/data-tracking.js b/packages/gatsby/src/query/__tests__/data-tracking.js index bbf8b203e116d..27e3b676f35ac 100644 --- a/packages/gatsby/src/query/__tests__/data-tracking.js +++ b/packages/gatsby/src/query/__tests__/data-tracking.js @@ -51,6 +51,7 @@ jest.mock(`../../utils/cache-lmdb`, () => { get = jest.fn(() => Promise.resolve()) set = jest.fn(() => Promise.resolve()) }, + __esModule: true, } }) diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index b2ed9217bdcf9..6e52917f4b02e 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -14,8 +14,18 @@ import errorParser from "./error-parser" import { GraphQLRunner } from "./graphql-runner" import { IExecutionResult, PageContext } from "./types" import { pageDataExists, savePageQueryResult } from "../utils/page-data" - -const resultHashes = new Map() +import GatsbyCacheLmdb from "../utils/cache-lmdb" + +let resultHashCache: GatsbyCacheLmdb | undefined +function getResultHashCache(): GatsbyCacheLmdb { + if (!resultHashCache) { + resultHashCache = new GatsbyCacheLmdb({ + name: `query-result-hashes`, + encoding: `string`, + }).init() + } + return resultHashCache +} export interface IQueryJob { id: string @@ -171,12 +181,13 @@ export async function queryRunner( .update(resultJSON) .digest(`base64`) + const resultHashCache = getResultHashCache() if ( - resultHash !== resultHashes.get(queryJob.id) || + resultHash !== (await resultHashCache.get(queryJob.id)) || (queryJob.isPage && !pageDataExists(path.join(program.directory, `public`), queryJob.id)) ) { - resultHashes.set(queryJob.id, resultHash) + await resultHashCache.set(queryJob.id, resultHash) if (queryJob.isPage) { // We need to save this temporarily in cache because diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap index 5f3d00b4f6315..b9a56bbce7c11 100644 --- a/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap @@ -80,6 +80,7 @@ Map { exports[`Add pages allows you to add pages 1`] = ` Object { + "componentModified": false, "contextModified": false, "payload": Object { "component": "/whatever/index.js", @@ -124,6 +125,7 @@ Map { exports[`Add pages allows you to add pages with context 1`] = ` Object { + "componentModified": false, "contextModified": false, "payload": Object { "component": "/whatever/index.js", @@ -172,6 +174,7 @@ Map { exports[`Add pages allows you to add pages with matchPath 1`] = ` Object { + "componentModified": false, "contextModified": false, "payload": Object { "component": "/whatever/index.js", diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index 7d234dab4d5ea..e4e1e19a6483c 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -429,6 +429,8 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} const oldPage: Page = store.getState().pages.get(internalPage.path) const contextModified = !!oldPage && !_.isEqual(oldPage.context, internalPage.context) + const componentModified = + !!oldPage && !_.isEqual(oldPage.component, internalPage.component) const alternateSlashPath = page.path.endsWith(`/`) ? page.path.slice(0, -1) @@ -496,6 +498,7 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} ...actionOptions, type: `CREATE_PAGE`, contextModified, + componentModified, plugin, payload: internalPage, }, diff --git a/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts b/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts index 84944b57ab360..27c96762afe72 100644 --- a/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts +++ b/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts @@ -7,6 +7,12 @@ export const pendingPageDataWritesReducer = ( action: ActionsUnion ): IGatsbyState["pendingPageDataWrites"] => { switch (action.type) { + case `CREATE_PAGE`: + if (action.componentModified) { + state.pagePaths.add(action.payload.path) + } + + return state case `ADD_PENDING_PAGE_DATA_WRITE`: state.pagePaths.add(action.payload.path) return state @@ -15,6 +21,7 @@ export const pendingPageDataWritesReducer = ( for (const page of action.payload.pages) { state.pagePaths.add(page) } + return state } diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts index 7aa9b417efa79..43b5e3c4ccc18 100644 --- a/packages/gatsby/src/redux/types.ts +++ b/packages/gatsby/src/redux/types.ts @@ -707,6 +707,7 @@ export interface ICreatePageAction { payload: IGatsbyPage plugin?: IGatsbyPlugin contextModified?: boolean + componentModified?: boolean } export interface ICreateRedirectAction {