diff --git a/integration-tests/artifacts/__tests__/index.js b/integration-tests/artifacts/__tests__/index.js index 3a7273eef6e27..5f55451fe0914 100644 --- a/integration-tests/artifacts/__tests__/index.js +++ b/integration-tests/artifacts/__tests__/index.js @@ -14,6 +14,23 @@ const gatsbyBin = path.join(`node_modules`, `gatsby`, `cli.js`) const manifest = {} const filesToRevert = {} +let _CFLAGS_ = { + GATSBY_MAJOR: `4`, +} +if (process.env.COMPILER_OPTIONS) { + // COMPILER_OPTIONS syntax is key=value,key2=value2 + _CFLAGS_ = process.env.COMPILER_OPTIONS.split(`,`).reduce((acc, curr) => { + const [key, value] = curr.split(`=`) + + if (key) { + acc[key] = value + } + + return acc + }, _CFLAGS_) +} + +let SLICES_ENABLED = _CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES let exitCode function runGatsbyWithRunTestSetup(runNumber = 1) { @@ -694,101 +711,146 @@ describe(`Second run (different pages created, data changed)`, () => { assertNodeCorrectness(runNumber) }) -describe(`Third run (js change, all pages are recreated)`, () => { - const runNumber = 3 - - const expectedPages = [ - `/stale-pages/only-not-in-first`, - `/page-query-dynamic-3/`, - `/page-that-will-have-trailing-slash-removed`, - `/stale-pages/sometimes-i-have-trailing-slash-sometimes-i-dont`, - ] - - const unexpectedPages = [ - `/stale-pages/only-in-first/`, - `/page-query-dynamic-1/`, - `/page-query-dynamic-2/`, - `/stateful-page-not-recreated-in-third-run/`, - ] - - let changedFileOriginalContent - const changedFileAbspath = path.join( - process.cwd(), - `src`, - `pages`, - `gatsby-browser.js` - ) +describe( + SLICES_ENABLED + ? `Third run (js template change, just pages of that template are recreated, all pages are stitched)` + : `Third run (js change, all pages are recreated)`, + + () => { + const runNumber = 3 + + const expectedPagesToRemainFromPreviousBuild = [ + `/stale-pages/stable/`, + `/page-query-stable/`, + `/page-query-changing-but-not-invalidating-html/`, + `/static-query-result-tracking/stable/`, + `/static-query-result-tracking/rerun-query-but-dont-recreate-html/`, + `/page-that-will-have-trailing-slash-removed`, + ] + + const expectedPagesToBeGenerated = [ + // this is page that gets template change + `/gatsby-browser/`, + // those change happen on every build + `/page-query-dynamic-3/`, + `/stale-pages/sometimes-i-have-trailing-slash-sometimes-i-dont`, + `/changing-context/`, + ] + + const expectedPages = [ + // this page should remain from first build + ...expectedPagesToRemainFromPreviousBuild, + // those pages should have been (re)created + ...expectedPagesToBeGenerated, + ] + + const unexpectedPages = [ + `/stale-pages/only-in-first/`, + `/page-query-dynamic-1/`, + `/page-query-dynamic-2/`, + `/stateful-page-not-recreated-in-third-run/`, + ] + + let changedFileOriginalContent + const changedFileAbspath = path.join( + process.cwd(), + `src`, + `pages`, + `gatsby-browser.js` + ) - beforeAll(async () => { - // make change to some .js - changedFileOriginalContent = fs.readFileSync(changedFileAbspath, `utf-8`) - filesToRevert[changedFileAbspath] = changedFileOriginalContent + beforeAll(async () => { + // make change to some .js + changedFileOriginalContent = fs.readFileSync(changedFileAbspath, `utf-8`) + filesToRevert[changedFileAbspath] = changedFileOriginalContent - const newContent = changedFileOriginalContent.replace(/sad/g, `not happy`) + const newContent = changedFileOriginalContent.replace(/sad/g, `not happy`) - if (newContent === changedFileOriginalContent) { - throw new Error(`Test setup failed`) - } + if (newContent === changedFileOriginalContent) { + throw new Error(`Test setup failed`) + } - fs.writeFileSync(changedFileAbspath, newContent) - await runGatsbyWithRunTestSetup(runNumber)() - }) + fs.writeFileSync(changedFileAbspath, newContent) + await runGatsbyWithRunTestSetup(runNumber)() + }) - assertExitCode(runNumber) + assertExitCode(runNumber) - describe(`html files`, () => { - const type = `html` + describe(`html files`, () => { + const type = `html` - describe(`should have expected html files`, () => { - assertFileExistenceForPagePaths({ - pagePaths: expectedPages, - type, - shouldExist: true, + describe(`should have expected html files`, () => { + assertFileExistenceForPagePaths({ + pagePaths: expectedPages, + type, + shouldExist: true, + }) }) - }) - describe(`shouldn't have unexpected html files`, () => { - assertFileExistenceForPagePaths({ - pagePaths: unexpectedPages, - type, - shouldExist: false, + describe(`shouldn't have unexpected html files`, () => { + assertFileExistenceForPagePaths({ + pagePaths: unexpectedPages, + type, + shouldExist: false, + }) }) - }) - it(`should recreate all html files`, () => { - expect(manifest[runNumber].generated.sort()).toEqual( - manifest[runNumber].allPages.sort() - ) + if (SLICES_ENABLED) { + it(`should recreate only some html files`, () => { + expect(manifest[runNumber].generated.sort()).toEqual( + expectedPagesToBeGenerated.sort() + ) + }) + + it(`should stitch fragments back in all html files (browser bundle changed)`, () => { + expect(manifest[runNumber].stitched.sort()).toEqual( + manifest[runNumber].allPages.sort() + ) + }) + } else { + it(`should recreate all html files`, () => { + expect(manifest[runNumber].generated.sort()).toEqual( + manifest[runNumber].allPages.sort() + ) + }) + } }) - }) - describe(`page-data files`, () => { - const type = `page-data` + describe(`page-data files`, () => { + const type = `page-data` - describe(`should have expected page-data files`, () => { - assertFileExistenceForPagePaths({ - pagePaths: expectedPages, - type, - shouldExist: true, + describe(`should have expected page-data files`, () => { + assertFileExistenceForPagePaths({ + pagePaths: expectedPages, + type, + shouldExist: true, + }) }) - }) - describe(`shouldn't have unexpected page-data files`, () => { - assertFileExistenceForPagePaths({ - pagePaths: unexpectedPages, - type, - shouldExist: false, + describe(`shouldn't have unexpected page-data files`, () => { + assertFileExistenceForPagePaths({ + pagePaths: unexpectedPages, + type, + shouldExist: false, + }) }) }) - }) - // third run - we modify module used by both ssr and browser bundle - both bundles should change - assertWebpackBundleChanges({ browser: true, ssr: true, runNumber }) + if (SLICES_ENABLED) { + // third run - we modify template used by both ssr and browser bundle - global, shared SSR won't change + // as the change is localized in just one of templates, which in Gatsby 5 doesn't invalidate all html + // files anymore + assertWebpackBundleChanges({ browser: true, ssr: false, runNumber }) + } else { + // third run - we modify module used by both ssr and browser bundle - both bundles should change + assertWebpackBundleChanges({ browser: true, ssr: true, runNumber }) + } - assertHTMLCorrectness(runNumber) + assertHTMLCorrectness(runNumber) - assertNodeCorrectness(runNumber) -}) + assertNodeCorrectness(runNumber) + } +) describe(`Fourth run (gatsby-browser change - cache get invalidated)`, () => { const runNumber = 4 diff --git a/integration-tests/artifacts/gatsby-node.js b/integration-tests/artifacts/gatsby-node.js index 824bfa8f95deb..d136d7350030a 100644 --- a/integration-tests/artifacts/gatsby-node.js +++ b/integration-tests/artifacts/gatsby-node.js @@ -9,8 +9,9 @@ let changedBrowserCompilationHash let changedSsrCompilationHash let regeneratedPages = [] let deletedPages = [] +let stitchedPages = [] -exports.onPreInit = ({ emitter }) => { +exports.onPreInit = ({ emitter, store }) => { emitter.on(`SET_WEBPACK_COMPILATION_HASH`, action => { changedBrowserCompilationHash = action.payload }) @@ -28,6 +29,17 @@ exports.onPreInit = ({ emitter }) => { emitter.on(`HTML_REMOVED`, action => { deletedPages.push(action.payload) }) + + // this is last step before stitching slice html into page html + // we don't have action specific for stitching, so we just use this one + // to read state that determine which page htmls will be stitched + emitter.on(`SLICES_PROPS_REMOVE_STALE`, () => { + stitchedPages = [] + + for (const path of store.getState().html.pagesThatNeedToStitchSlices) { + stitchedPages.push(path) + } + }) } let previouslyCreatedNodes = new Map() @@ -215,6 +227,7 @@ exports.onPreBuild = () => { changedSsrCompilationHash = `not-changed` regeneratedPages = [] deletedPages = [] + stitchedPages = [] } let counter = 1 @@ -250,6 +263,7 @@ exports.onPostBuild = async ({ graphql, getNodes }) => { changedSsrCompilationHash, generated: regeneratedPages, removed: deletedPages, + stitched: stitchedPages, } ) } diff --git a/integration-tests/ssr/test-output.js b/integration-tests/ssr/test-output.js index 09e75301fe719..77c3b60091478 100644 --- a/integration-tests/ssr/test-output.js +++ b/integration-tests/ssr/test-output.js @@ -33,6 +33,15 @@ async function run() { // Only in dev $(`meta[name="note"]`).remove() + // remove any comments + $.root() + .find("*") + .contents() + .filter(function () { + return this.type === "comment" + }) + .remove() + return $.html() } diff --git a/packages/gatsby-cli/src/reporter/loggers/ink/components/pageTree.tsx b/packages/gatsby-cli/src/reporter/loggers/ink/components/pageTree.tsx index 3bac89d3e6233..eab4ac2fc806a 100644 --- a/packages/gatsby-cli/src/reporter/loggers/ink/components/pageTree.tsx +++ b/packages/gatsby-cli/src/reporter/loggers/ink/components/pageTree.tsx @@ -13,6 +13,7 @@ import { interface IPageTreeProps { components: Map root: string + slices: Set } const Description: React.FC = function Description(props) { @@ -84,6 +85,7 @@ const ComponentTree: React.FC<{ const PageTree: React.FC = function PageTree({ components, root, + slices, }) { const componentList: Array = [] let i = 0 @@ -107,6 +109,20 @@ const PageTree: React.FC = function PageTree({ Pages {componentList} + {slices.size > 0 && ( + <> + + Slices + + {Array.from(slices).map(slice => ( + + + · {path.posix.relative(root, slice)} + + + ))} + + )} ) @@ -116,8 +132,13 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() { const state = useContext(StoreStateContext) const componentWithPages = new Map() + const slices = new Set() - for (const { componentPath, pages } of state.pageTree!.components.values()) { + for (const { + componentPath, + pages, + isSlice, + } of state.pageTree!.components.values()) { const layoutComponent = getPathToLayoutComponent(componentPath) const pagesByMode = componentWithPages.get(layoutComponent) || { SSG: new Set(), @@ -125,12 +146,17 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() { SSR: new Set(), FN: new Set(), } - pages.forEach(pagePath => { - const gatsbyPage = state.pageTree!.pages.get(pagePath) - pagesByMode[gatsbyPage!.mode].add(pagePath) - }) - componentWithPages.set(layoutComponent, pagesByMode) + if (isSlice) { + slices.add(componentPath) + } else { + pages.forEach(pagePath => { + const gatsbyPage = state.pageTree!.pages.get(pagePath) + + pagesByMode[gatsbyPage!.mode].add(pagePath) + }) + componentWithPages.set(layoutComponent, pagesByMode) + } } for (const { @@ -146,7 +172,11 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() { } return ( - + ) } diff --git a/packages/gatsby-cli/src/reporter/loggers/yurnalist/index.ts b/packages/gatsby-cli/src/reporter/loggers/yurnalist/index.ts index 8ae03dda2fc0f..69dda19a46dfd 100644 --- a/packages/gatsby-cli/src/reporter/loggers/yurnalist/index.ts +++ b/packages/gatsby-cli/src/reporter/loggers/yurnalist/index.ts @@ -34,7 +34,8 @@ function generatePageTreeToConsole( ): void { const root = state.root const componentWithPages = new Map() - for (const { componentPath, pages } of state.components.values()) { + const slices = new Set() + for (const { componentPath, pages, isSlice } of state.components.values()) { const layoutComponent = getPathToLayoutComponent(componentPath) const relativePath = path.posix.relative(root, layoutComponent) const pagesByMode = componentWithPages.get(relativePath) || { @@ -43,13 +44,18 @@ function generatePageTreeToConsole( SSR: new Set(), FN: new Set(), } - pages.forEach(pagePath => { - const gatsbyPage = state.pages.get(pagePath) - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - pagesByMode[gatsbyPage!.mode].add(pagePath) - }) - componentWithPages.set(relativePath, pagesByMode) + if (isSlice) { + slices.add(path.posix.relative(root, componentPath)) + } else { + pages.forEach(pagePath => { + const gatsbyPage = state.pages.get(pagePath) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + pagesByMode[gatsbyPage!.mode].add(pagePath) + }) + + componentWithPages.set(relativePath, pagesByMode) + } } for (const { @@ -98,6 +104,15 @@ function generatePageTreeToConsole( i++ } + if (slices.size > 0) { + pageTreeConsole.push(``) + pageTreeConsole.push(`\n${chalk.underline(`Slices`)}\n`) + + Array.from(slices).forEach(slice => { + pageTreeConsole.push(`· ${slice}`) + }) + } + pageTreeConsole.push(``) pageTreeConsole.push( boxen( diff --git a/packages/gatsby-cli/src/reporter/types.ts b/packages/gatsby-cli/src/reporter/types.ts index 951e446de2480..7992960439c3f 100644 --- a/packages/gatsby-cli/src/reporter/types.ts +++ b/packages/gatsby-cli/src/reporter/types.ts @@ -63,6 +63,7 @@ type PageMode = "SSG" | "DSG" | "SSR" interface IGatsbyPageComponent { componentPath: string pages: Set + isSlice: boolean } interface IGatsbyPage { mode: PageMode diff --git a/packages/gatsby-cli/src/structured-errors/error-map.ts b/packages/gatsby-cli/src/structured-errors/error-map.ts index f432f7d810f97..78083eaf577a8 100644 --- a/packages/gatsby-cli/src/structured-errors/error-map.ts +++ b/packages/gatsby-cli/src/structured-errors/error-map.ts @@ -433,9 +433,10 @@ const errors = { context.pageObject, null, 4 - )}\n\nSee the documentation for the "createPage" action — https://www.gatsbyjs.com/docs/reference/config-files/actions#createPage`, + )}`, level: Level.ERROR, category: ErrorCategory.USER, + docsUrl: `https://www.gatsbyjs.com/docs/reference/config-files/actions#createPage`, }, "11323": { text: (context): string => @@ -445,9 +446,10 @@ const errors = { context.pageObject, null, 4 - )}\n\nSee the documentation for the "createPage" action — https://www.gatsbyjs.com/docs/reference/config-files/actions#createPage`, + )}`, level: Level.ERROR, category: ErrorCategory.USER, + docsUrl: `https://www.gatsbyjs.com/docs/reference/config-files/actions#createPage`, }, "11324": { text: (context): string => @@ -565,6 +567,36 @@ const errors = { category: ErrorCategory.USER, docsUrl: `https://www.gatsbyjs.com/docs/reference/functions/`, }, + // slices + "11333": { + text: (context): string => + `${ + context.pluginName + } created a slice and didn't pass the path to the component.\n\nThe slice object passed to createSlice:\n${JSON.stringify( + context.sliceObject, + null, + 4 + )}`, + level: Level.ERROR, + category: ErrorCategory.USER, + // TODO: change domain to gatsbyjs.com when it's released + docsUrl: `https://v5.gatsbyjs.com/docs/reference/config-files/actions#createSlice`, + }, + "11334": { + text: (context): string => + `${ + context.pluginName + } must set the slice id when creating a slice.\n\nThe slice object passed to createSlice:\n${JSON.stringify( + context.sliceObject, + null, + 4 + )}`, + level: Level.ERROR, + category: ErrorCategory.USER, + // TODO: change domain to gatsbyjs.com when it's released + docsUrl: `https://v5.gatsbyjs.com/docs/reference/config-files/actions#createSlice`, + }, + // node object didn't pass validation "11467": { text: (context): string => diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/dev-loader.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/dev-loader.js.snap index 34dd744801c1b..8451a9b5badf8 100644 --- a/packages/gatsby/cache-dir/__tests__/__snapshots__/dev-loader.js.snap +++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/dev-loader.js.snap @@ -16,6 +16,7 @@ Object { "getServerDataError": undefined, "matchPath": undefined, "path": "/mypage/", + "slicesMap": Object {}, "staticQueryHashes": Array [], "webpackCompilationHash": "123", }, diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/loader.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/loader.js.snap index ac23c5f70eeb4..a715d888745dd 100644 --- a/packages/gatsby/cache-dir/__tests__/__snapshots__/loader.js.snap +++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/loader.js.snap @@ -12,6 +12,7 @@ Object { "getServerDataError": undefined, "matchPath": undefined, "path": "/mypage/", + "slicesMap": Object {}, "staticQueryHashes": Array [], "webpackCompilationHash": "123", }, diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap index d0ac0ceb5a74a..31017c8bea06d 100644 --- a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap +++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap @@ -15,6 +15,7 @@ exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComp exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = ` Object { "html": "
", + "sliceData": Object {}, "unsafeBuiltinsUsage": Array [], } `; @@ -22,6 +23,7 @@ Object { exports[`static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = ` Object { "html": "
div3
div2
div1
", + "sliceData": Object {}, "unsafeBuiltinsUsage": Array [], } `; @@ -29,6 +31,7 @@ Object { exports[`static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = ` Object { "html": "
div3
div2
div1
", + "sliceData": Object {}, "unsafeBuiltinsUsage": Array [], } `; diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js index 0deec9c83fbf0..f30071ae26a29 100644 --- a/packages/gatsby/cache-dir/__tests__/static-entry.js +++ b/packages/gatsby/cache-dir/__tests__/static-entry.js @@ -64,6 +64,7 @@ const pageDataMock = { componentChunkName: `page-component---src-pages-test-js`, path: `/about/`, staticQueryHashes: [], + slicesMap: new Map(), } const webpackCompilationHash = `1234567890abcdef1234` @@ -371,6 +372,7 @@ describe(`static-entry sanity checks`, () => { styles: [], reversedStyles: [], reversedScripts: [], + sliceData: {}, } beforeEach(() => { diff --git a/packages/gatsby/cache-dir/create-content-digest-browser-shim.js b/packages/gatsby/cache-dir/create-content-digest-browser-shim.js new file mode 100644 index 0000000000000..edc673f1095b6 --- /dev/null +++ b/packages/gatsby/cache-dir/create-content-digest-browser-shim.js @@ -0,0 +1 @@ +exports.createContentDigest = () => `` diff --git a/packages/gatsby/cache-dir/dev-loader.js b/packages/gatsby/cache-dir/dev-loader.js index 7cc77f262653d..f831b94f57586 100644 --- a/packages/gatsby/cache-dir/dev-loader.js +++ b/packages/gatsby/cache-dir/dev-loader.js @@ -55,6 +55,8 @@ class DevLoader extends BaseLoader { this.handleStaticQueryResultHotUpdate(msg) } else if (msg.type === `pageQueryResult`) { this.handlePageQueryResultHotUpdate(msg) + } else if (msg.type === `sliceQueryResult`) { + this.handleSliceQueryResultHotUpdate(msg) } else if (msg.type === `stalePageData`) { this.handleStalePageDataMessage(msg) } else if (msg.type === `staleServerData`) { @@ -119,6 +121,37 @@ class DevLoader extends BaseLoader { } } + handleSliceQueryResultHotUpdate(msg) { + const newResult = msg.payload.result + + const cacheKey = msg.payload.id + + // raw json db + { + const cachedResult = this.slicesDataDb.get(cacheKey) + if (!isEqual(newResult, cachedResult)) { + this.slicesDataDb.set(cacheKey, newResult) + } + } + + // processed data + { + const cachedResult = this.slicesDb.get(cacheKey) + if ( + !isEqual(newResult?.result?.data, cachedResult?.data) || + !isEqual(newResult?.result?.sliceContext, cachedResult?.sliceContext) + ) { + const mergedResult = { + ...cachedResult, + data: newResult?.result?.data, + sliceContext: newResult?.result?.sliceContext, + } + this.slicesDb.set(cacheKey, mergedResult) + ___emitter.emit(`sliceQueryResult`, mergedResult) + } + } + } + updatePageData = (pagePath, newPageData) => { const pageDataDbCacheKey = normalizePagePath(pagePath) const cachedPageData = this.pageDataDb.get(pageDataDbCacheKey)?.payload diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 2caf7c85f648e..976431cab30ad 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -17,6 +17,8 @@ export const PageResourceStatus = { Success: `success`, } +const preferDefault = m => (m && m.default) || m + const stripSurroundingSlashes = s => { s = s[0] === `/` ? s.slice(1) : s s = s.endsWith(`/`) ? s.slice(0, -1) : s @@ -70,6 +72,7 @@ const toPageResources = (pageData, component = null, head) => { matchPath: pageData.matchPath, staticQueryHashes: pageData.staticQueryHashes, getServerDataError: pageData.getServerDataError, + slicesMap: pageData.slicesMap ?? {}, } return { @@ -122,6 +125,9 @@ export class BaseLoader { this.staticQueryDb = {} this.pageDataDb = new Map() this.partialHydrationDb = new Map() + this.slicesDataDb = new Map() + this.sliceInflightDb = new Map() + this.slicesDb = new Map() this.isPrefetchQueueRunning = false this.prefetchQueued = [] this.prefetchTriggered = new Set() @@ -315,6 +321,21 @@ export class BaseLoader { }) } + loadSliceDataJson(sliceName) { + if (this.slicesDataDb.has(sliceName)) { + const jsonPayload = this.slicesDataDb.get(sliceName) + return Promise.resolve({ sliceName, jsonPayload }) + } + + const url = `/slice-data/${sliceName}.json` + return doFetch(url, `GET`).then(res => { + const jsonPayload = JSON.parse(res.responseText) + + this.slicesDataDb.set(sliceName, jsonPayload) + return { sliceName, jsonPayload } + }) + } + findMatchPath(rawPath) { return findMatchPath(rawPath) } @@ -487,104 +508,168 @@ export class BaseLoader { } let pageData = result.payload - const { componentChunkName, staticQueryHashes = [] } = pageData + const { + componentChunkName, + staticQueryHashes: pageStaticQueryHashes = [], + slicesMap = {}, + } = pageData const finalResult = {} - // In develop we have separate chunks for template and Head components - // to enable HMR (fast refresh requires single exports). - // In production we have shared chunk with both exports. Double loadComponent here - // will be deduped by webpack runtime resulting in single request and single module - // being loaded for both `component` and `head`. - const componentChunkPromise = Promise.all([ - this.loadComponent(componentChunkName), - this.loadComponent(componentChunkName, `head`), - ]).then(([component, head]) => { - finalResult.createdAt = new Date() - let pageResources - if (!component || component instanceof Error) { - finalResult.status = PageResourceStatus.Error - finalResult.error = component - } else { - finalResult.status = PageResourceStatus.Success - if (result.notFound === true) { - finalResult.notFound = true - } - pageData = Object.assign(pageData, { - webpackCompilationHash: allData[0] - ? allData[0].webpackCompilationHash - : ``, - }) - pageResources = toPageResources(pageData, component, head) + const dedupedSliceNames = Array.from(new Set(Object.values(slicesMap))) + + const loadSlice = slice => { + if (this.slicesDb.has(slice.name)) { + return this.slicesDb.get(slice.name) + } else if (this.sliceInflightDb.has(slice.name)) { + return this.sliceInflightDb.get(slice.name) } - // undefined if final result is an error - return pageResources - }) - const staticQueryBatchPromise = Promise.all( - staticQueryHashes.map(staticQueryHash => { - // Check for cache in case this static query result has already been loaded - if (this.staticQueryDb[staticQueryHash]) { - const jsonPayload = this.staticQueryDb[staticQueryHash] - return { staticQueryHash, jsonPayload } + const inFlight = this.loadComponent(slice.componentChunkName).then( + component => { + return { + component: preferDefault(component), + sliceContext: slice.result.sliceContext, + data: slice.result.data, + } } + ) - return this.memoizedGet( - `${__PATH_PREFIX__}/page-data/sq/d/${staticQueryHash}.json` - ) - .then(req => { - const jsonPayload = JSON.parse(req.responseText) - return { staticQueryHash, jsonPayload } - }) - .catch(() => { - throw new Error( - `We couldn't load "${__PATH_PREFIX__}/page-data/sq/d/${staticQueryHash}.json"` - ) - }) + this.sliceInflightDb.set(slice.name, inFlight) + inFlight.then(results => { + this.slicesDb.set(slice.name, results) + this.sliceInflightDb.delete(slice.name) }) - ).then(staticQueryResults => { - const staticQueryResultsMap = {} - staticQueryResults.forEach(({ staticQueryHash, jsonPayload }) => { - staticQueryResultsMap[staticQueryHash] = jsonPayload - this.staticQueryDb[staticQueryHash] = jsonPayload - }) + return inFlight + } - return staticQueryResultsMap - }) + return Promise.all( + dedupedSliceNames.map(sliceName => this.loadSliceDataJson(sliceName)) + ).then(slicesData => { + const slices = [] + const dedupedStaticQueryHashes = [...pageStaticQueryHashes] + for (const { jsonPayload, sliceName } of Object.values(slicesData)) { + slices.push({ name: sliceName, ...jsonPayload }) + for (const staticQueryHash of jsonPayload.staticQueryHashes) { + if (!dedupedStaticQueryHashes.includes(staticQueryHash)) { + dedupedStaticQueryHashes.push(staticQueryHash) + } + } + } - return ( - Promise.all([componentChunkPromise, staticQueryBatchPromise]) - .then(([pageResources, staticQueryResults]) => { - let payload - if (pageResources) { - payload = { ...pageResources, staticQueryResults } - finalResult.payload = payload - emitter.emit(`onPostLoadPageResources`, { - page: payload, - pageResources: payload, - }) + // In develop we have separate chunks for template and Head components + // to enable HMR (fast refresh requires single exports). + // In production we have shared chunk with both exports. Double loadComponent here + // will be deduped by webpack runtime resulting in single request and single module + // being loaded for both `component` and `head`. + // get list of components to get + const componentChunkPromises = Promise.all([ + this.loadComponent(componentChunkName), + this.loadComponent(componentChunkName, `head`), + Promise.all(slices.map(loadSlice)), + ]).then(components => { + const [rootComponent, headComponent, sliceComponents] = components + finalResult.createdAt = new Date() + for (const sliceComponent of sliceComponents) { + if (!sliceComponent || sliceComponent instanceof Error) { + finalResult.status = PageResourceStatus.Error + finalResult.error = sliceComponent } + } - this.pageDb.set(pagePath, finalResult) + if (!rootComponent || rootComponent instanceof Error) { + finalResult.status = PageResourceStatus.Error + finalResult.error = rootComponent + } - if (finalResult.error) { - return { - error: finalResult.error, - status: finalResult.status, - } + let pageResources + if (finalResult.status !== PageResourceStatus.Error) { + finalResult.status = PageResourceStatus.Success + if (result.notFound === true) { + finalResult.notFound = true } + pageData = Object.assign(pageData, { + webpackCompilationHash: allData[0] + ? allData[0].webpackCompilationHash + : ``, + }) + pageResources = toPageResources( + pageData, + rootComponent, + headComponent + ) + } + // undefined if final result is an error + return pageResources + }) - return payload - }) - // when static-query fail to load we throw a better error - .catch(err => { - return { - error: err, - status: PageResourceStatus.Error, + // get list of static queries to get + const staticQueryBatchPromise = Promise.all( + dedupedStaticQueryHashes.map(staticQueryHash => { + // Check for cache in case this static query result has already been loaded + if (this.staticQueryDb[staticQueryHash]) { + const jsonPayload = this.staticQueryDb[staticQueryHash] + return { staticQueryHash, jsonPayload } } + + return this.memoizedGet( + `${__PATH_PREFIX__}/page-data/sq/d/${staticQueryHash}.json` + ) + .then(req => { + const jsonPayload = JSON.parse(req.responseText) + return { staticQueryHash, jsonPayload } + }) + .catch(() => { + throw new Error( + `We couldn't load "${__PATH_PREFIX__}/page-data/sq/d/${staticQueryHash}.json"` + ) + }) }) - ) + ).then(staticQueryResults => { + const staticQueryResultsMap = {} + + staticQueryResults.forEach(({ staticQueryHash, jsonPayload }) => { + staticQueryResultsMap[staticQueryHash] = jsonPayload + this.staticQueryDb[staticQueryHash] = jsonPayload + }) + + return staticQueryResultsMap + }) + + return ( + Promise.all([componentChunkPromises, staticQueryBatchPromise]) + .then(([pageResources, staticQueryResults]) => { + let payload + if (pageResources) { + payload = { ...pageResources, staticQueryResults } + finalResult.payload = payload + emitter.emit(`onPostLoadPageResources`, { + page: payload, + pageResources: payload, + }) + } + + this.pageDb.set(pagePath, finalResult) + + if (finalResult.error) { + return { + error: finalResult.error, + status: finalResult.status, + } + } + + return payload + }) + // when static-query fail to load we throw a better error + .catch(err => { + return { + error: err, + status: PageResourceStatus.Error, + } + }) + ) + }) }) } @@ -948,3 +1033,11 @@ export function getStaticQueryResults() { return {} } } + +export function getSliceResults() { + if (instance) { + return instance.slicesDb + } else { + return {} + } +} diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index 7a5847bd9c606..eb047fbc4f0fd 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -3,6 +3,11 @@ import React from "react" import { Router, navigate, Location, BaseContext } from "@gatsbyjs/reach-router" import { ScrollContext } from "gatsby-react-router-scroll" import { StaticQueryContext } from "gatsby" +import { + SlicesMapContext, + SlicesContext, + SlicesResultsContext, +} from "./slice/context" import { shouldUpdateScroll, init as navigationInit, @@ -17,6 +22,7 @@ import { publicLoader, PageResourceStatus, getStaticQueryResults, + getSliceResults, } from "./loader" import EnsureResources from "./ensure-resources" import stripPrefix from "./strip-prefix" @@ -67,6 +73,10 @@ apiRunnerAsync(`onClientEntry`).then(() => { const DataContext = React.createContext({}) + const slicesContext = { + renderEnvironment: `browser`, + } + class GatsbyRoot extends React.Component { render() { const { children } = this.props @@ -83,11 +93,22 @@ apiRunnerAsync(`onClientEntry`).then(() => { ) } else { const staticQueryResults = getStaticQueryResults() + const sliceResults = getSliceResults() return ( - - {children} - + + + + + {children} + + + + ) } diff --git a/packages/gatsby/cache-dir/query-result-store.js b/packages/gatsby/cache-dir/query-result-store.js index 8f15e11da7d8b..3f4fc66cc9a25 100644 --- a/packages/gatsby/cache-dir/query-result-store.js +++ b/packages/gatsby/cache-dir/query-result-store.js @@ -6,7 +6,8 @@ import { } from "./socketIo" import PageRenderer from "./page-renderer" import normalizePagePath from "./normalize-page-path" -import loader, { getStaticQueryResults } from "./loader" +import loader, { getStaticQueryResults, getSliceResults } from "./loader" +import { SlicesResultsContext } from "./slice/context" if (process.env.NODE_ENV === `production`) { throw new Error( @@ -154,3 +155,43 @@ export class StaticQueryStore extends React.Component { ) } } + +export class SliceDataStore extends React.Component { + constructor(props) { + super(props) + this.state = { + slicesData: new Map(getSliceResults()), + } + } + + handleMittEvent = () => { + this.setState({ + slicesData: new Map(getSliceResults()), + }) + } + + componentDidMount() { + ___emitter.on(`sliceQueryResult`, this.handleMittEvent) + ___emitter.on(`onPostLoadPageResources`, this.handleMittEvent) + } + + componentWillUnmount() { + ___emitter.off(`sliceQueryResult`, this.handleMittEvent) + ___emitter.off(`onPostLoadPageResources`, this.handleMittEvent) + } + + shouldComponentUpdate(nextProps, nextState) { + // We want to update this component when: + // - slice results changed + + return this.state.slicesData !== nextState.slicesData + } + + render() { + return ( + + {this.props.children} + + ) + } +} diff --git a/packages/gatsby/cache-dir/root.js b/packages/gatsby/cache-dir/root.js index 66c034d7641dd..e9bf2eda942c6 100644 --- a/packages/gatsby/cache-dir/root.js +++ b/packages/gatsby/cache-dir/root.js @@ -2,10 +2,15 @@ import React from "react" import { Router, Location, BaseContext } from "@gatsbyjs/reach-router" import { ScrollContext } from "gatsby-react-router-scroll" +import { SlicesMapContext, SlicesContext } from "./slice/context" import { shouldUpdateScroll, RouteUpdates } from "./navigation" import { apiRunner } from "./api-runner-browser" import loader from "./loader" -import { PageQueryStore, StaticQueryStore } from "./query-result-store" +import { + PageQueryStore, + StaticQueryStore, + SliceDataStore, +} from "./query-result-store" import EnsureResources from "./ensure-resources" import FastRefreshOverlay from "./fast-refresh-overlay" @@ -32,33 +37,44 @@ class LocationHandler extends React.Component { render() { const { location } = this.props + const slicesContext = { + renderEnvironment: `browser`, + } + if (!loader.isPageNotFound(location.pathname + location.search)) { return ( {locationAndPageResources => ( - - + - - - - - + + + + + + + + + )} ) @@ -111,7 +127,9 @@ const rootWrappedWithWrapRootElement = apiRunner( function RootWrappedWithOverlayAndProvider() { return ( - {rootWrappedWithWrapRootElement} + + {rootWrappedWithWrapRootElement} + ) } diff --git a/packages/gatsby/cache-dir/slice.js b/packages/gatsby/cache-dir/slice.js new file mode 100644 index 0000000000000..a286e323907f5 --- /dev/null +++ b/packages/gatsby/cache-dir/slice.js @@ -0,0 +1,114 @@ +import React, { useContext } from "react" +import { ServerSlice } from "./slice/server-slice" +import { InlineSlice } from "./slice/inline-slice" +import { SlicesContext } from "./slice/context" + +export function Slice(props) { + if (process.env.GATSBY_SLICES) { + // we use sliceName internally, so remap alias to sliceName + const internalProps = { + ...props, + sliceName: props.alias, + } + delete internalProps.alias + + const slicesContext = useContext(SlicesContext) + + // validate props + const propErrors = validateSliceProps(props) + if (Object.keys(propErrors).length) { + throw new SlicePropsError( + slicesContext.renderEnvironment === `browser`, + internalProps.sliceName, + propErrors + ) + } + + if (slicesContext.renderEnvironment === `server`) { + return + } else if (slicesContext.renderEnvironment === `browser`) { + // in the browser, we'll just render the component as is + return + } else if (slicesContext.renderEnvironment === `engines`) { + // if we're in SSR, we'll just render the component as is + return + } else { + throw new Error( + `Slice context "${slicesContext.renderEnvironment}" is not supported.` + ) + } + } else { + throw new Error( + `Slices are disabled, likely due to PARTIAL_HYDRATION flag being set.` + ) + } +} + +class SlicePropsError extends Error { + constructor(inBrowser, sliceName, propErrors) { + const errors = Object.entries(propErrors) + .map(([key, value]) => `${key}: "${value}"`) + .join(`, `) + + const name = `SlicePropsError` + let stack = `` + let message = `` + + if (inBrowser) { + // They're just (kinda) kidding, I promise... You can still work here <3 + // https://www.gatsbyjs.com/careers/ + const fullStack = + React.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED.ReactDebugCurrentFrame.getCurrentStack() + + // remove the first line of the stack trace + const stackLines = fullStack.trim().split(`\n`).slice(1) + stackLines[0] = stackLines[0].trim() + stack = `\n` + stackLines.join(`\n`) + + // look for any hints for the component name in the stack trace + const componentRe = /^at\s+([a-zA-Z0-9]+)/ + const componentMatch = stackLines[0].match(componentRe) + const componentHint = componentMatch ? `in ${componentMatch[1]} ` : `` + + message = `Slice "${sliceName}" was passed props ${componentHint}that are not serializable (${errors}).` + } else { + // we can't really grab any extra info outside of the browser, so just print what we can + message = `${name}: Slice "${sliceName}" was passed props that are not serializable (${errors}). Use \`gatsby develop\` to see more information.` + const stackLines = new Error().stack.trim().split(`\n`).slice(2) + stack = `${message}\n${stackLines.join(`\n`)}` + } + + super(message) + this.name = name + this.stack = stack + } +} + +const validateSliceProps = ( + props, + errors = {}, + seenObjects = [], + path = null +) => { + // recursively validate all props + for (const [name, value] of Object.entries(props)) { + if ( + value === undefined || + value === null || + (!path && name === `children`) + ) { + continue + } + + const propPath = path ? `${path}.${name}` : name + + if (typeof value === `function`) { + errors[propPath] = typeof value + } else if (typeof value === `object` && seenObjects.indexOf(value) <= 0) { + seenObjects.push(value) + validateSliceProps(value, errors, seenObjects, propPath) + } + } + + return errors +} diff --git a/packages/gatsby/cache-dir/slice/context.js b/packages/gatsby/cache-dir/slice/context.js new file mode 100644 index 0000000000000..cb592584938f9 --- /dev/null +++ b/packages/gatsby/cache-dir/slice/context.js @@ -0,0 +1,13 @@ +import React from "react" + +const SlicesResultsContext = React.createContext({}) +const SlicesContext = React.createContext({}) +const SlicesMapContext = React.createContext({}) +const SlicesPropsContext = React.createContext({}) + +export { + SlicesResultsContext, + SlicesContext, + SlicesMapContext, + SlicesPropsContext, +} diff --git a/packages/gatsby/cache-dir/slice/inline-slice.js b/packages/gatsby/cache-dir/slice/inline-slice.js new file mode 100644 index 0000000000000..7e7f8dc507813 --- /dev/null +++ b/packages/gatsby/cache-dir/slice/inline-slice.js @@ -0,0 +1,34 @@ +import React, { useContext } from "react" +import { SlicesMapContext, SlicesResultsContext } from "./context" + +export const InlineSlice = ({ + sliceName, + allowEmpty, + children, + ...sliceProps +}) => { + const slicesMap = useContext(SlicesMapContext) + const slicesResultsMap = useContext(SlicesResultsContext) + const concreteSliceName = slicesMap[sliceName] + const slice = slicesResultsMap.get(concreteSliceName) + + if (!slice) { + if (allowEmpty) { + return null + } else { + throw new Error( + `Slice "${concreteSliceName}" for "${sliceName}" slot not found` + ) + } + } + + return ( + + {children} + + ) +} diff --git a/packages/gatsby/cache-dir/slice/server-slice-renderer.js b/packages/gatsby/cache-dir/slice/server-slice-renderer.js new file mode 100644 index 0000000000000..53a9b36f7335f --- /dev/null +++ b/packages/gatsby/cache-dir/slice/server-slice-renderer.js @@ -0,0 +1,28 @@ +import React from "react" + +export const ServerSliceRenderer = ({ sliceId, children }) => { + const contents = [ + React.createElement(`slice-start`, { + id: `${sliceId}-1`, + }), + React.createElement(`slice-end`, { + id: `${sliceId}-1`, + }), + ] + + if (children) { + // if children exist, we split the slice into a before and after piece + // see renderSlices in render-html + contents.push(children) + contents.push( + React.createElement(`slice-start`, { + id: `${sliceId}-2`, + }), + React.createElement(`slice-end`, { + id: `${sliceId}-2`, + }) + ) + } + + return contents +} diff --git a/packages/gatsby/cache-dir/slice/server-slice.js b/packages/gatsby/cache-dir/slice/server-slice.js new file mode 100644 index 0000000000000..4fe811ee0adfa --- /dev/null +++ b/packages/gatsby/cache-dir/slice/server-slice.js @@ -0,0 +1,52 @@ +import React, { useContext } from "react" +import { createContentDigest } from "gatsby-core-utils/create-content-digest" +import { SlicesMapContext, SlicesPropsContext } from "./context" +import { ServerSliceRenderer } from "./server-slice-renderer" + +const getSliceId = (sliceName, sliceProps) => { + if (!Object.keys(sliceProps).length) { + return sliceName + } + + const propsString = createContentDigest(sliceProps) + return `${sliceName}-${propsString}` +} + +export const ServerSlice = ({ + sliceName, + allowEmpty, + children, + ...sliceProps +}) => { + const slicesMap = useContext(SlicesMapContext) + const slicesProps = useContext(SlicesPropsContext) + const concreteSliceName = slicesMap[sliceName] + + if (!concreteSliceName) { + if (allowEmpty) { + return null + } else { + throw new Error( + `Slice "${concreteSliceName}" for "${sliceName}" slot not found` + ) + } + } + + const sliceId = getSliceId(concreteSliceName, sliceProps) + + // set props on context object for static-entry to return + let sliceUsage = slicesProps[sliceId] + if (!sliceUsage) { + slicesProps[sliceId] = sliceUsage = { + props: sliceProps, + sliceName: concreteSliceName, + hasChildren: !!children, + } + } else { + if (children) { + sliceUsage.hasChildren = true + } + } + + return {children} +} diff --git a/packages/gatsby/cache-dir/static-entry.js b/packages/gatsby/cache-dir/static-entry.js index aa1be92b0d90e..dbed9219a7b91 100644 --- a/packages/gatsby/cache-dir/static-entry.js +++ b/packages/gatsby/cache-dir/static-entry.js @@ -17,8 +17,14 @@ const { apiRunner, apiRunnerAsync } = require(`./api-runner-ssr`) const asyncRequires = require(`$virtual/async-requires`) const { version: gatsbyVersion } = require(`gatsby/package.json`) const { grabMatchParams } = require(`./find-path`) -const chunkMapping = require(`../public/chunk-map.json`) const { headHandlerForSSR } = require(`./head/head-export-handler-for-ssr`) +const { + SlicesResultsContext, + SlicesContext, + SlicesMapContext, + SlicesPropsContext, +} = require(`./slice/context`) +const { ServerSliceRenderer } = require(`./slice/server-slice-renderer`) // we want to force posix-style joins, so Windows doesn't produce backslashes for urls const { join } = path.posix @@ -105,6 +111,12 @@ export const reorderHeadComponents = headComponents => { return sorted } +const DEFAULT_CONTEXT = { + // whether or not we're building the site now + // usage in determining original build or engines + isDuringBuild: false, +} + export default async function staticPage({ pagePath, pageData, @@ -114,8 +126,12 @@ export default async function staticPage({ reversedStyles, reversedScripts, inlinePageData = false, + context = {}, webpackCompilationHash, + sliceData, }) { + const renderContext = Object.assign(DEFAULT_CONTEXT, context) + // for this to work we need this function to be sync or at least ensure there is single execution of it at a time global.unsafeBuiltinUsage = [] @@ -210,7 +226,7 @@ export default async function staticPage({ postBodyComponents = sanitizeComponents(components) } - const { componentChunkName } = pageData + const { componentChunkName, slicesMap } = pageData const pageComponent = await asyncRequires.components[componentChunkName]() headHandlerForSSR({ @@ -256,16 +272,64 @@ export default async function staticPage({ ) + const sliceProps = {} + + let body = apiRunner( + `wrapRootElement`, + { element: routerElement, pathname: pagePath }, + routerElement, + ({ result }) => { + return { element: result, pathname: pagePath } + } + ).pop() + + if (process.env.GATSBY_SLICES) { + const slicesContext = { + // if we're in build now, we know we're on the server + // otherwise we're in an engine + renderEnvironment: renderContext.isDuringBuild ? `server` : `engines`, + } + // if we're running in an engine, we need to manually wrap body with + // the results context to pass the map of slice name to component/data/context + if (slicesContext.renderEnvironment === `engines`) { + // this is the same name used in the browser + // since this immitates behavior + const slicesDb = new Map() + + for (const sliceName of Object.values(slicesMap)) { + const slice = sliceData[sliceName] + const { default: SliceComponent } = await getPageChunk(slice) + + const sliceObject = { + component: SliceComponent, + sliceContext: slice.result.sliceContext, + data: slice.result.data, + } + + slicesDb.set(sliceName, sliceObject) + } + + body = ( + + {body} + + ) + } + + body = ( + + + + {body} + + + + ) + } + const bodyComponent = ( - {apiRunner( - `wrapRootElement`, - { element: routerElement, pathname: pagePath }, - routerElement, - ({ result }) => { - return { element: result, pathname: pagePath } - } - ).pop()} + {body} ) @@ -364,7 +428,11 @@ export default async function staticPage({ }) // Add page metadata for the current page - const windowPageData = `/**/` @@ -378,46 +446,55 @@ export default async function staticPage({ /> ) - // Add chunk mapping metadata - const scriptChunkMapping = `/**/` - - postBodyComponents.push( - + ` + + await fs.ensureDir(path.join(`public`, `_gatsby`, `slices`)) + + const hashSliceContents = `` + + const assetSliceContents: Array = [] + + if (`polyfill` in assets && assets.polyfill) { + for (const asset of assets.polyfill) { + if (asset.endsWith(`.js`)) { + assetSliceContents.push( + `` + ) + } + } + } + + if (`app` in assets && assets.app) { + for (const asset of assets.app) { + if (asset.endsWith(`.js`)) { + assetSliceContents.push( + `` + ) + } + } + } + + const scriptsSliceHtmlChanged = await ensureFileContent( + path.join(`public`, `_gatsby`, `slices`, `_gatsby-scripts-1.html`), + chunkSliceContents + hashSliceContents + assetSliceContents.join(``) + ) + + if (scriptsSliceHtmlChanged) { + store.dispatch({ + type: `SLICES_SCRIPTS_REGENERATED`, + }) + } + } + previousChunkMapJson = newChunkMapJson } diff --git a/packages/gatsby/src/utils/js-chunk-names.ts b/packages/gatsby/src/utils/js-chunk-names.ts index 90b152406008c..b78dbba120db9 100644 --- a/packages/gatsby/src/utils/js-chunk-names.ts +++ b/packages/gatsby/src/utils/js-chunk-names.ts @@ -39,7 +39,10 @@ function replaceUnifiedRoutesKeys( } const chunkNameCache = new Map() -export function generateComponentChunkName(componentPath: string): string { +export function generateComponentChunkName( + componentPath: string, + kind: "component" | "slice" = `component` +): string { if (chunkNameCache.has(componentPath)) { return chunkNameCache.get(componentPath) } else { @@ -65,7 +68,7 @@ export function generateComponentChunkName(componentPath: string): string { name = `${hash}-${name.substring(name.length - 60)}` } - const chunkName = `component---${name}` + const chunkName = `${kind}---${name}` chunkNameCache.set(componentPath, chunkName) diff --git a/packages/gatsby/src/utils/page-data-helpers.ts b/packages/gatsby/src/utils/page-data-helpers.ts index 64d28f1a385cb..8f1a9d56736ab 100644 --- a/packages/gatsby/src/utils/page-data-helpers.ts +++ b/packages/gatsby/src/utils/page-data-helpers.ts @@ -1,32 +1,119 @@ +import reporter from "gatsby-cli/lib/reporter" import type { IStructuredError } from "gatsby-cli/src/structured-errors/types" -import { IGatsbyPage } from "../redux/types" +import { IGatsbyPage, IGatsbyState } from "../redux/types" +import { ICollectedSlices } from "./babel/find-slices" -export interface IPageData { +interface IPageDataBase { componentChunkName: IGatsbyPage["componentChunkName"] - matchPath?: IGatsbyPage["matchPath"] + matchPath: IGatsbyPage["matchPath"] path: IGatsbyPage["path"] staticQueryHashes: Array getServerDataError?: IStructuredError | Array | null manifestId?: string } +export type IPageDataInput = IPageDataBase & { + slices: Record + componentPath: string +} + +export type IPageData = IPageDataBase & { + slicesMap: Record +} + +function traverseSlicesUsedByTemplates( + pagePath: string, + componentPath: string, + overrideSlices: Record, + slicesUsedByTemplates: Map, + slices: IGatsbyState["slices"], + formattedSlices: Record = {}, + handledSlices: Set = new Set() +): Record | null { + if (handledSlices.has(componentPath)) { + return null + } else { + handledSlices.add(componentPath) + } + + const slicesUsedByComponent = slicesUsedByTemplates.get(componentPath) + if (!slicesUsedByComponent) { + return null + } + + for (const [sliceSlot, sliceConf] of Object.entries(slicesUsedByComponent)) { + let concreteSliceForSliceSlot = sliceSlot + + if (overrideSlices && overrideSlices[sliceSlot]) { + concreteSliceForSliceSlot = overrideSlices[sliceSlot] + } + + const slice = slices.get(concreteSliceForSliceSlot) + if (!slice) { + if (sliceConf.allowEmpty) { + continue + } else { + const message = + `Could not find slice "${concreteSliceForSliceSlot}" used by page "${pagePath}". ` + + `Please check your createPages in your gatsby-node to verify this ` + + `is the correct name or set allowEmpty to true.` + + reporter.panicOnBuild(new Error(message)) + continue + } + } + + formattedSlices[sliceSlot] = concreteSliceForSliceSlot + + // recursively repeat for found slice to find all nested slices + traverseSlicesUsedByTemplates( + pagePath, + slice.componentPath, + overrideSlices, + slicesUsedByTemplates, + slices, + formattedSlices, + handledSlices + ) + } + + return formattedSlices +} export function constructPageDataString( { componentChunkName, + componentPath, matchPath, path: pagePath, staticQueryHashes, manifestId, - }: IPageData, - result: string | Buffer + slices: overrideSlices, + }: IPageDataInput, + result: string | Buffer, + slicesUsedByTemplates: Map, + slices: IGatsbyState["slices"] ): string { let body = `{` + `"componentChunkName":"${componentChunkName}",` + - `"path":${JSON.stringify(pagePath)},` + + (pagePath ? `"path":${JSON.stringify(pagePath)},` : ``) + `"result":${result},` + `"staticQueryHashes":${JSON.stringify(staticQueryHashes)}` + if (_CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES) { + const formattedSlices = traverseSlicesUsedByTemplates( + pagePath, + componentPath, + overrideSlices, + slicesUsedByTemplates, + slices + ) + + if (formattedSlices) { + body += `,"slicesMap":${JSON.stringify(formattedSlices)}` + } + } + if (matchPath) { body += `,"matchPath":"${matchPath}"` } diff --git a/packages/gatsby/src/utils/page-data.ts b/packages/gatsby/src/utils/page-data.ts index a1f957a626a24..21dc295228959 100644 --- a/packages/gatsby/src/utils/page-data.ts +++ b/packages/gatsby/src/utils/page-data.ts @@ -7,6 +7,7 @@ import { createContentDigest, generatePageDataPath } from "gatsby-core-utils" import { websocketManager } from "./websocket-manager" import { isWebpackStatusPending } from "./webpack-status" import { store } from "../redux" +import type { IGatsbySlice, IGatsbyState } from "../redux/types" import { hasFlag, FLAG_DIRTY_NEW_PAGE } from "../redux/reducers/queries" import { isLmdbStore } from "../datastore" import type GatsbyCacheLmdb from "./cache-lmdb" @@ -14,6 +15,7 @@ import { constructPageDataString, reverseFixedPagePath, IPageData, + IPageDataInput, } from "./page-data-helpers" import { Span } from "opentracing" @@ -21,6 +23,8 @@ export { reverseFixedPagePath } import { processNodeManifests } from "../utils/node-manifest" import { IExecutionResult } from "../query/types" import { getPageMode } from "./page-mode" +import { ICollectedSlices } from "./babel/find-slices" +import { ensureFileContent } from "./ensure-file-content" export interface IPageDataWithQueryResult extends IPageData { result: IExecutionResult @@ -32,7 +36,6 @@ export async function readPageData( ): Promise { const filePath = generatePageDataPath(publicDir, pagePath) const rawPageData = await fs.readFile(filePath, `utf-8`) - return JSON.parse(rawPageData) } @@ -116,13 +119,20 @@ export async function readPageQueryResult( export async function writePageData( publicDir: string, - pageData: IPageData + pageData: IPageDataInput, + slicesUsedByTemplates: Map, + slices: IGatsbyState["slices"] ): Promise { const result = await readPageQueryResult(publicDir, pageData.path) const outputFilePath = generatePageDataPath(publicDir, pageData.path) - const body = constructPageDataString(pageData, result) + const body = constructPageDataString( + pageData, + result, + slicesUsedByTemplates, + slices + ) // transform asset size to kB (from bytes) to fit 64 bit to numbers const pageDataSize = Buffer.byteLength(body) / 1000 @@ -137,10 +147,52 @@ export async function writePageData( }, }) - await fs.outputFile(outputFilePath, body) + await ensureFileContent(outputFilePath, body) return body } +export async function writeSliceData( + publicDir: string, + { componentChunkName, name }: IGatsbySlice, + staticQueryHashes: Array +): Promise { + const result = JSON.parse( + (await readPageQueryResult(publicDir, `slice--${name}`)).toString() + ) + + const outputFilePath = path.join(publicDir, `slice-data`, `${name}.json`) + + const body = JSON.stringify({ + componentChunkName, + result, + staticQueryHashes, + }) + + const sliceDataSize = Buffer.byteLength(body) / 1000 + + store.dispatch({ + type: `ADD_SLICE_DATA_STATS`, + payload: { + sliceName: name, + filePath: outputFilePath, + size: sliceDataSize, + sliceDataHash: createContentDigest(body), + }, + }) + + await ensureFileContent(outputFilePath, body) + return body +} + +export async function readSliceData( + publicDir: string, + sliceName: string +): Promise { + const filePath = path.join(publicDir, `slice-data`, `${sliceName}.json`) + const rawPageData = await fs.readFile(filePath, `utf-8`) + return JSON.parse(rawPageData) +} + let isFlushPending = false let isFlushing = false @@ -148,6 +200,16 @@ export function isFlushEnqueued(): boolean { return isFlushPending } +type IDataTask = + | { + type: "page" + pagePath: string + } + | { + type: "slice" + sliceName: string + } + export async function flush(parentSpan?: Span): Promise { if (isFlushing) { // We're already in the middle of a flush @@ -162,10 +224,12 @@ export async function flush(parentSpan?: Span): Promise { program, staticQueriesByTemplate, queries, + slices, + slicesByTemplate, } = store.getState() const isBuild = program?._?.[0] !== `develop` - const { pagePaths } = pendingPageDataWrites + const { pagePaths, sliceNames } = pendingPageDataWrites let writePageDataActivity let nodeManifestPagePathMap @@ -175,81 +239,128 @@ export async function flush(parentSpan?: Span): Promise { // We use this manifestId to determine if the page data is up to date when routing. Here we create a map of "pagePath": "manifestId" while processing and writing node manifest files. // We only do this when there are pending page-data writes because otherwise we could flush pending createNodeManifest calls before page-data.json files are written. Which means those page-data files wouldn't have the corresponding manifest id's written to them. nodeManifestPagePathMap = await processNodeManifests() + } + if (pagePaths.size > 0 || sliceNames.size > 0) { writePageDataActivity = reporter.createProgress( - `Writing page-data.json files to public directory`, - pagePaths.size, + `Writing page-data.json and slice-data.json files to public directory`, + pagePaths.size + sliceNames.size, 0, { id: `write-page-data-public-directory`, parentSpan } ) writePageDataActivity.start() } - const flushQueue = fastq(async (pagePath, cb) => { - const page = pages.get(pagePath) - - // It's a gloomy day in Bombay, let me tell you a short story... - // Once upon a time, writing page-data.json files were atomic - // After this change (#24808), they are not and this means that - // between adding a pending write for a page and actually flushing - // them, a page might not exist anymore щ（ﾟДﾟщ） - // This is why we need this check - if (page) { - if (page.path && nodeManifestPagePathMap) { - page.manifestId = nodeManifestPagePathMap.get(page.path) - } + const flushQueue = fastq(async (task, cb) => { + if (task.type === `page`) { + const { pagePath } = task + const page = pages.get(pagePath) + + let shouldClearPendingWrite = true + + // It's a gloomy day in Bombay, let me tell you a short story... + // Once upon a time, writing page-data.json files were atomic + // After this change (#24808), they are not and this means that + // between adding a pending write for a page and actually flushing + // them, a page might not exist anymore щ（ﾟДﾟщ） + // This is why we need this check + if (page) { + if (page.path && nodeManifestPagePathMap) { + page.manifestId = nodeManifestPagePathMap.get(page.path) + } - if (!isBuild && process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) { - // check if already did run query for this page - // with query-on-demand we might have pending page-data write due to - // changes in static queries assigned to page template, but we might not - // have query result for it - const query = queries.trackedQueries.get(page.path) - if (!query) { - // this should not happen ever - throw new Error( - `We have a page, but we don't have registered query for it (???)` - ) + if (!isBuild && process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) { + // check if already did run query for this page + // with query-on-demand we might have pending page-data write due to + // changes in static queries assigned to page template, but we might not + // have query result for it + const query = queries.trackedQueries.get(page.path) + if (!query) { + // this should not happen ever + throw new Error( + `We have a page, but we don't have registered query for it (???)` + ) + } + + if (hasFlag(query.dirty, FLAG_DIRTY_NEW_PAGE)) { + // query results are not written yet + setImmediate(() => cb(null, true)) + return + } } - if (hasFlag(query.dirty, FLAG_DIRTY_NEW_PAGE)) { - // query results are not written yet - setImmediate(() => cb(null, true)) - return + // In develop we rely on QUERY_ON_DEMAND so we just go through + // In build we only build these page-json for SSG pages + if (!isBuild || (isBuild && getPageMode(page) === `SSG`)) { + const staticQueryHashes = + staticQueriesByTemplate.get(page.componentPath) || [] + + try { + const result = await writePageData( + path.join(program.directory, `public`), + { + ...page, + staticQueryHashes, + }, + slicesByTemplate, + slices + ) + + if (!isBuild) { + websocketManager.emitPageData({ + id: pagePath, + result: JSON.parse(result) as IPageDataWithQueryResult, + }) + } + } catch (e) { + shouldClearPendingWrite = false + reporter.panicOnBuild( + `Failed to write page-data for ""${page.path}`, + e + ) + } + writePageDataActivity.tick() } } - // In develop we rely on QUERY_ON_DEMAND so we just go through - // In build we only build these page-json for SSG pages - if (!isBuild || (isBuild && getPageMode(page) === `SSG`)) { + if (shouldClearPendingWrite) { + store.dispatch({ + type: `CLEAR_PENDING_PAGE_DATA_WRITE`, + payload: { + page: pagePath, + }, + }) + } + } else if (task.type === `slice`) { + const { sliceName } = task + const slice = slices.get(sliceName) + if (slice) { const staticQueryHashes = - staticQueriesByTemplate.get(page.componentPath) || [] + staticQueriesByTemplate.get(slice.componentPath) || [] - const result = await writePageData( + const result = await writeSliceData( path.join(program.directory, `public`), - { - ...page, - staticQueryHashes, - } + slice, + staticQueryHashes ) writePageDataActivity.tick() if (!isBuild) { - websocketManager.emitPageData({ - id: pagePath, + websocketManager.emitSliceData({ + id: sliceName, result: JSON.parse(result) as IPageDataWithQueryResult, }) } } - } - store.dispatch({ - type: `CLEAR_PENDING_PAGE_DATA_WRITE`, - payload: { - page: pagePath, - }, - }) + store.dispatch({ + type: `CLEAR_PENDING_SLICE_DATA_WRITE`, + payload: { + name: sliceName, + }, + }) + } // `setImmediate` below is a workaround against stack overflow // occurring when there are many non-SSG pages @@ -258,7 +369,10 @@ export async function flush(parentSpan?: Span): Promise { }, 25) for (const pagePath of pagePaths) { - flushQueue.push(pagePath, () => {}) + flushQueue.push({ type: `page`, pagePath }, () => {}) + } + for (const sliceName of sliceNames) { + flushQueue.push({ type: `slice`, sliceName }, () => {}) } if (!flushQueue.idle()) { diff --git a/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts b/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts index b00bd594cbca8..adcd8b9b60e6c 100644 --- a/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts +++ b/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts @@ -1,4 +1,5 @@ import * as path from "path" +import * as fs from "fs-extra" import webpack from "webpack" import mod from "module" import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging" @@ -11,6 +12,7 @@ import { } from "../client-assets-for-template" import { writeStaticQueryContext } from "../static-query-utils" import { IGatsbyState } from "../../redux/types" +import { store } from "../../redux" type Reporter = typeof reporter @@ -56,6 +58,22 @@ export async function createPageSSRBundle({ reporter: Reporter isVerbose?: boolean }): Promise { + const state = store.getState() + const slicesStateObject = {} + for (const [key, value] of state.slices) { + slicesStateObject[key] = value + } + + const slicesByTemplateStateObject = {} + for (const [template, records] of state.slicesByTemplate) { + const recordsObject = {} + for (const path of Object.keys(records)) { + recordsObject[path] = records[path] + } + + slicesByTemplateStateObject[template] = recordsObject + } + const webpackStats = await readWebpackStats(path.join(rootDir, `public`)) const toInline: Record = {} @@ -153,6 +171,22 @@ export async function createPageSSRBundle({ new webpack.DefinePlugin({ INLINED_TEMPLATE_TO_DETAILS: JSON.stringify(toInline), WEBPACK_COMPILATION_HASH: JSON.stringify(webpackCompilationHash), + GATSBY_SLICES: JSON.stringify(slicesStateObject), + GATSBY_SLICES_BY_TEMPLATE: JSON.stringify(slicesByTemplateStateObject), + GATSBY_SLICES_SCRIPT: JSON.stringify( + _CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES + ? fs.readFileSync( + path.join( + rootDir, + `public`, + `_gatsby`, + `slices`, + `_gatsby-scripts-1.html` + ), + `utf-8` + ) + : `` + ), // eslint-disable-next-line @typescript-eslint/naming-convention "process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`), }), diff --git a/packages/gatsby/src/utils/page-ssr-module/entry.ts b/packages/gatsby/src/utils/page-ssr-module/entry.ts index 488ffb46f3b5e..1448ed490dd4c 100644 --- a/packages/gatsby/src/utils/page-ssr-module/entry.ts +++ b/packages/gatsby/src/utils/page-ssr-module/entry.ts @@ -5,7 +5,7 @@ import "../engines-fs-provider" // just types - those should not be bundled import type { GraphQLEngine } from "../../schema/graphql-engine/entry" import type { IExecutionResult } from "../../query/types" -import type { IGatsbyPage } from "../../redux/types" +import type { IGatsbyPage, IGatsbySlice, IGatsbyState } from "../../redux/types" import { IGraphQLTelemetryRecord } from "../../schema/type-definitions" import type { IScriptsAndStyles } from "../client-assets-for-template" import type { IPageDataWithQueryResult } from "../page-data" @@ -25,6 +25,7 @@ import { getServerData, IServerData } from "../get-server-data" import reporter from "gatsby-cli/lib/reporter" import { initTracer } from "../tracer" import { getCodeFrame } from "../../query/graphql-errors-codeframe" +import { ICollectedSlice } from "../babel/find-slices" export interface ITemplateDetails { query: string @@ -46,6 +47,7 @@ export interface ISSRData { declare global { const INLINED_TEMPLATE_TO_DETAILS: Record const WEBPACK_COMPILATION_HASH: string + const GATSBY_SLICES_SCRIPT: string } const tracerReadyPromise = initTracer( @@ -264,14 +266,45 @@ export async function renderPageData({ }) activity.start() } + + const componentPath = data.page.componentPath + const sliceOverrides = data.page.slices + + // @ts-ignore GATSBY_SLICES is being "inlined" by bundler + const slicesFromBundler = GATSBY_SLICES as { + [key: string]: IGatsbySlice + } + const slices: IGatsbyState["slices"] = new Map() + for (const [key, value] of Object.entries(slicesFromBundler)) { + slices.set(key, value) + } + + const slicesUsedByTemplatesFromBundler = + // @ts-ignore GATSBY_SLICES_BY_TEMPLATE is being "inlined" by bundler + GATSBY_SLICES_BY_TEMPLATE as { + [key: string]: { [key: string]: ICollectedSlice } + } + const slicesUsedByTemplates: IGatsbyState["slicesByTemplate"] = new Map() + for (const [key, value] of Object.entries( + slicesUsedByTemplatesFromBundler + )) { + slicesUsedByTemplates.set(key, value) + } + + // TODO: optimize this to only pass name for slices, as it's only used for validation + const results = await constructPageDataString( { componentChunkName: data.page.componentChunkName, path: getPath(data), matchPath: data.page.matchPath, staticQueryHashes: data.templateDetails.staticQueryHashes, + componentPath, + slices: sliceOverrides, }, - JSON.stringify(data.results) + JSON.stringify(data.results), + slicesUsedByTemplates, + slices ) return JSON.parse(results) @@ -296,6 +329,15 @@ const readStaticQueryContext = async ( return JSON.parse(rawSQContext) } +const readSliceData = async ( + sliceName: string +): Promise> => { + const filePath = path.join(__dirname, `slice-data`, `${sliceName}.json`) + + const rawSliceData = await fs.readFile(filePath, `utf-8`) + return JSON.parse(rawSliceData) +} + export async function renderHTML({ data, pageData, @@ -344,6 +386,13 @@ export async function renderHTML({ } } + const sliceData = {} + if (_CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES) { + for (const sliceName of Object.values(pageData.slicesMap)) { + sliceData[sliceName] = await readSliceData(sliceName) + } + } + let renderHTMLActivity: MaybePhantomActivity try { if (wrapperActivity) { @@ -356,16 +405,21 @@ export async function renderHTML({ renderHTMLActivity.start() } + const pagePath = getPath(data) const results = await htmlComponentRenderer({ - pagePath: getPath(data), + pagePath, pageData, staticQueryContext, webpackCompilationHash: WEBPACK_COMPILATION_HASH, ...data.templateDetails.assets, inlinePageData: data.page.mode === `SSR` && data.results.serverData, + sliceData, }) - return results.html + return results.html.replace( + ``, + GATSBY_SLICES_SCRIPT + ) } finally { if (renderHTMLActivity) { renderHTMLActivity.end() diff --git a/packages/gatsby/src/utils/slices.ts b/packages/gatsby/src/utils/slices.ts new file mode 100644 index 0000000000000..0530a5b41142f --- /dev/null +++ b/packages/gatsby/src/utils/slices.ts @@ -0,0 +1,6 @@ +import type { IGatsbySlice } from "../redux/types" +import { createContentDigest } from "gatsby-core-utils/create-content-digest" + +export function getSliceId(slice: IGatsbySlice): string { + return `${slice.componentChunkName}-${createContentDigest(slice.context)}` +} diff --git a/packages/gatsby/src/utils/slices/stitching.ts b/packages/gatsby/src/utils/slices/stitching.ts new file mode 100644 index 0000000000000..3e6eb5443af44 --- /dev/null +++ b/packages/gatsby/src/utils/slices/stitching.ts @@ -0,0 +1,141 @@ +import * as path from "path" +import * as fs from "fs-extra" +import { generateHtmlPath } from "gatsby-core-utils/page-html" + +interface ISliceBoundaryMatch { + index: number + end: number + syntax: "element" | "comment" + id: string + type: "start" | "end" +} + +function ensureExpectedType(maybeType: string): "start" | "end" { + if (maybeType === `start` || maybeType === `end`) { + return maybeType + } else { + throw new Error(`Unexpected type: ${maybeType}. Expected "start" or "end"`) + } +} + +async function stitchSlices( + htmlString: string, + publicDir: string +): Promise { + let previousStart: ISliceBoundaryMatch | undefined = undefined + + let processedHTML = `` + let cursor = 0 + + async function getSliceContent(sliceHtmlName: string): Promise { + return fs.readFile( + path.join(publicDir, `_gatsby`, `slices`, `${sliceHtmlName}.html`), + `utf-8` + ) + } + + for (const match of htmlString.matchAll( + /(start|end)\s[^>]*id="(?[^"]+)"[^>]*><\/slice-(?[^>]+)>|)/g + )) { + if (!match.groups) { + throw new Error( + `Invariant: [stitching slices] Capturing groups should be defined` + ) + } + + if (typeof match.index !== `number`) { + throw new Error( + `Invariant: [stitching slices] There is no location of a match when stitching slices` + ) + } + + if ( + match.groups.startOrEndElementOpenening && + match.groups.startOrEndElementOpenening !== + match.groups.startOrEndElementClosing + ) { + throw new Error( + `Invariant: [stitching slices] start and end tags should be the same. Got: Start: ${match.groups.startOrEndElementOpenening} End: ${match.groups.startOrEndElementClosing}` + ) + } + + const meta: ISliceBoundaryMatch = { + index: match.index, + end: match.index + match[0].length, + ...(match.groups.startOrEndElementOpenening + ? { + syntax: `element`, // can discard this field + id: match.groups.idElement, + type: ensureExpectedType(match.groups.startOrEndElementOpenening), + } + : { + syntax: `comment`, // can discard this field + id: match.groups.idComment, + type: ensureExpectedType(match.groups.startOrEndComment), + }), + } + + if (meta.type === `start`) { + if (previousStart) { + // if we are already in a slice, we will replace everything until the outer slice end + // so we just ignore those + continue + } + const newCursor = meta.end + processedHTML += + htmlString.substring(cursor, meta.index) + + `` + cursor = newCursor + + previousStart = meta + } else if (meta.type === `end`) { + if (!previousStart) { + throw new Error( + `Invariant: [stitching slices] There was no start tag, but close tag was found` + ) + } + if (previousStart.id !== meta.id) { + // it's possible to have nested slices - we want to handle just the most outer slice + // as stitching it in will recursively handle nested slices as well + continue + } + + processedHTML += `${await stitchSlices( + await getSliceContent(meta.id), + publicDir + )}` + cursor = meta.end + + previousStart = undefined + } + } + + if (previousStart) { + throw new Error( + `Invariant: [stitching slices] There was start tag, but no close tag was found` + ) + } + + // get rest of the html + processedHTML += htmlString.substring(cursor) + + return processedHTML +} + +export async function stitchSliceForAPage({ + pagePath, + publicDir, +}: { + pagePath: string + publicDir: string +}): Promise { + const htmlFilePath = generateHtmlPath(publicDir, pagePath) + + const html = await fs.readFile(htmlFilePath, `utf-8`) + + const processedHTML = await stitchSlices(html, publicDir) + + if (html !== processedHTML) { + await fs.writeFile(htmlFilePath, processedHTML) + } +} diff --git a/packages/gatsby/src/utils/webpack-utils.ts b/packages/gatsby/src/utils/webpack-utils.ts index d010ac1af8fac..087ae4b1dfc90 100644 --- a/packages/gatsby/src/utils/webpack-utils.ts +++ b/packages/gatsby/src/utils/webpack-utils.ts @@ -13,6 +13,7 @@ import { getBrowsersList } from "./browserslist" import ESLintPlugin from "eslint-webpack-plugin" import { cpuCoreCount } from "gatsby-core-utils" import { GatsbyWebpackStatsExtractor } from "./gatsby-webpack-stats-extractor" +import { getPublicPath } from "./get-public-path" import { GatsbyWebpackVirtualModules, VIRTUAL_MODULES_BASE_PATH, @@ -182,6 +183,9 @@ export const createWebpackUtils = ( const isSSR = stage.includes(`html`) const { config } = store.getState() + const { assetPrefix, pathPrefix } = config + + const publicPath = getPublicPath({ assetPrefix, pathPrefix, ...program }) const makeExternalOnly = (original: RuleFactory) => @@ -782,7 +786,7 @@ export const createWebpackUtils = ( plugins.ignore({ resourceRegExp: /^\.\/locale$/, contextRegExp: /moment$/ }) plugins.extractStats = (): GatsbyWebpackStatsExtractor => - new GatsbyWebpackStatsExtractor() + new GatsbyWebpackStatsExtractor(publicPath) // TODO: remove this in v5 plugins.eslintGraphqlSchemaReload = (): null => null diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js index 2360e27d49d63..8d9f04a997b5f 100644 --- a/packages/gatsby/src/utils/webpack.config.js +++ b/packages/gatsby/src/utils/webpack.config.js @@ -500,16 +500,20 @@ module.exports = async ( ], } - // TODO(v5): Remove since this is only useful during Gatsby 4 publishes - if (_CFLAGS_.GATSBY_MAJOR !== `5`) { - const target = - stage === `build-html` || stage === `develop-html` ? `node` : `web` - if (target === `web`) { + const target = + stage === `build-html` || stage === `develop-html` ? `node` : `web` + if (target === `web`) { + // TODO(v5): Remove since this is only useful during Gatsby 4 publishes + if (_CFLAGS_.GATSBY_MAJOR !== `5`) { resolve.alias[`@reach/router`] = path.join( getPackageRoot(`@gatsbyjs/reach-router`), `es` ) } + + resolve.alias[`gatsby-core-utils/create-content-digest`] = directoryPath( + `.cache/create-content-digest-browser-shim` + ) } if (stage === `build-javascript` && program.profile) { diff --git a/packages/gatsby/src/utils/webpack/get-ssr-chunk-hashes.ts b/packages/gatsby/src/utils/webpack/get-ssr-chunk-hashes.ts new file mode 100644 index 0000000000000..9af7c56da994e --- /dev/null +++ b/packages/gatsby/src/utils/webpack/get-ssr-chunk-hashes.ts @@ -0,0 +1,74 @@ +import type webpack from "webpack" +import type { IGatsbyState } from "../../internal" + +type ChunkGroup = webpack.Compilation["chunkGroups"][0] + +function getHashes( + chunkGroup: ChunkGroup, + compilation: webpack.Compilation, + hashes: Array = [] +): Array { + for (const chunk of chunkGroup.chunks) { + if (!chunk.hash) { + throw new Error( + `Invariant: [generating template hashes] Chunk doesn't have hash` + ) + } + hashes.push(chunk.hash) + } + + for (const childChunkGroups of Object.values( + chunkGroup.getChildrenByOrders( + compilation.moduleGraph, + compilation.chunkGraph + ) + )) { + for (const childChunkGroup of childChunkGroups) { + getHashes(childChunkGroup, compilation, hashes) + } + } + return hashes +} + +export function getSSRChunkHashes({ + stats, + components, +}: { + stats: webpack.Stats + components: IGatsbyState["components"] +}): { + templateHashes: Record + renderPageHash: string +} { + const templateHashes: Record = {} + const componentChunkNameToTemplatePath: Record = {} + let renderPageHash = `` + + components.forEach(component => { + componentChunkNameToTemplatePath[component.componentChunkName] = + component.componentPath + }) + + for (const chunkGroup of stats.compilation.chunkGroups) { + if (chunkGroup.name) { + const concenatedChunksByName = getHashes( + chunkGroup, + stats.compilation + ).join(`--`) + + if (chunkGroup.name !== `render-page`) { + const templatePath = componentChunkNameToTemplatePath[chunkGroup.name] + if (!templatePath) { + // additional chunk group can be created by lazy imports + // we handle them by handling children chunk groups on top level ones + continue + } + templateHashes[templatePath] = concenatedChunksByName + } else { + renderPageHash = concenatedChunksByName + } + } + } + + return { templateHashes, renderPageHash } +} diff --git a/packages/gatsby/src/utils/webpack/plugins/static-query-mapper.ts b/packages/gatsby/src/utils/webpack/plugins/static-query-mapper.ts index b27d46ca4bc46..d8cb63be39c98 100644 --- a/packages/gatsby/src/utils/webpack/plugins/static-query-mapper.ts +++ b/packages/gatsby/src/utils/webpack/plugins/static-query-mapper.ts @@ -2,7 +2,7 @@ import path from "path" import { Store } from "redux" import { Compiler, Module, NormalModule, Compilation } from "webpack" import ConcatenatedModule from "webpack/lib/optimize/ConcatenatedModule" -import { isEqual } from "lodash" +import { isEqual, cloneDeep } from "lodash" import { generateComponentChunkName } from "../../js-chunk-names" import { enqueueFlush } from "../../page-data" import type { @@ -10,6 +10,10 @@ import type { IGatsbyPageComponent, IGatsbyStaticQueryComponents, } from "../../../redux/types" +import { + ICollectedSlice, + mergePreviouslyCollectedSlices, +} from "../../babel/find-slices" type ChunkGroup = Compilation["chunkGroups"][0] type EntryPoint = Compilation["asyncEntrypoints"][0] @@ -79,14 +83,23 @@ function getRealPath( function getWebpackModulesByResourcePaths( modules: Set, staticQueries: IGatsbyState["staticQueryComponents"], - components: IGatsbyState["components"] + components: IGatsbyState["components"], + componentsUsingSlices: IGatsbyState["componentsUsingSlices"] ): { webpackModulesByStaticQueryId: Map> webpackModulesByComponentId: Map> + webpackModulesUsingSlices: Set<{ + module: Module + slices: Record + }> } { const realPathCache = new Map() const webpackModulesByStaticQueryId = new Map>() const webpackModulesByComponentId = new Map>() + const webpackModulesUsingSlices = new Set<{ + module: Module + slices: Record + }>() modules.forEach(webpackModule => { for (const [id, staticQuery] of staticQueries) { @@ -131,9 +144,25 @@ function getWebpackModulesByResourcePaths( webpackModulesByComponentId.set(id, set) } + + for (const [filePath, slices] of componentsUsingSlices) { + const componentComponentPath = getRealPath(realPathCache, filePath) + if (!doesModuleMatchResourcePath(componentComponentPath, webpackModule)) { + continue + } + + webpackModulesUsingSlices.add({ + module: webpackModule, + slices: slices, + }) + } }) - return { webpackModulesByStaticQueryId, webpackModulesByComponentId } + return { + webpackModulesByStaticQueryId, + webpackModulesByComponentId, + webpackModulesUsingSlices, + } } /** @@ -172,7 +201,8 @@ export class StaticQueryMapper { } apply(compiler: Compiler): void { - const { components, staticQueryComponents } = this.store.getState() + const { components, staticQueryComponents, componentsUsingSlices } = + this.store.getState() compiler.hooks.done.tap(this.name, stats => { const compilation = stats.compilation @@ -183,18 +213,26 @@ export class StaticQueryMapper { } const staticQueriesByChunkGroup = new Map>() + const pageSliceUsageByChunkGroup = new Map< + ChunkGroup, + Record + >() const chunkGroupsWithPageComponents = new Set() const chunkGroupsByComponentPath = new Map< IGatsbyPageComponent["componentPath"], ChunkGroup >() - const { webpackModulesByStaticQueryId, webpackModulesByComponentId } = - getWebpackModulesByResourcePaths( - compilation.modules, - staticQueryComponents, - components - ) + const { + webpackModulesByStaticQueryId, + webpackModulesByComponentId, + webpackModulesUsingSlices, + } = getWebpackModulesByResourcePaths( + compilation.modules, + staticQueryComponents, + components, + componentsUsingSlices + ) const appEntryPoint = ( compilation.entrypoints.has(`app`) @@ -246,6 +284,39 @@ export class StaticQueryMapper { }) } + // group Slice usage by chunkGroup for ease of use + for (const { + slices, + module: webpackModule, + } of webpackModulesUsingSlices) { + let chunkGroupsDerivedFromEntrypoints: Array = [] + for (const chunk of compilation.chunkGraph.getModuleChunksIterable( + webpackModule + )) { + for (const chunkGroup of chunk.groupsIterable) { + if (chunkGroup === appEntryPoint) { + chunkGroupsDerivedFromEntrypoints.push(chunkGroup) + } else { + chunkGroupsDerivedFromEntrypoints = + chunkGroupsDerivedFromEntrypoints.concat( + getChunkGroupsDerivedFromEntrypoint(chunkGroup, appEntryPoint) + ) + } + } + } + + // loop over all component chunkGroups or global ones + chunkGroupsDerivedFromEntrypoints.forEach(chunkGroup => { + pageSliceUsageByChunkGroup.set( + chunkGroup, + mergePreviouslyCollectedSlices( + slices, + pageSliceUsageByChunkGroup.get(chunkGroup) + ) + ) + }) + } + // group chunkGroups by componentPaths for ease of use for (const [ componentPath, @@ -276,41 +347,105 @@ export class StaticQueryMapper { } } + let globalSliceUsage: Record = {} + for (const [chunkGroup, slices] of pageSliceUsageByChunkGroup) { + if ( + !chunkGroupsWithPageComponents.has(chunkGroup) && + !chunkGroup.name?.endsWith(`head`) + ) { + globalSliceUsage = mergePreviouslyCollectedSlices( + slices, + globalSliceUsage + ) + } + } + components.forEach(component => { const allStaticQueries = new Set(globalStaticQueries) + // we only add global slices to pages, not other slices + let allSlices: Record = component.isSlice + ? {} + : cloneDeep(globalSliceUsage) + if (chunkGroupsByComponentPath.has(component.componentPath)) { const chunkGroup = chunkGroupsByComponentPath.get( component.componentPath ) - if (chunkGroup && staticQueriesByChunkGroup.has(chunkGroup)) { - ;( - staticQueriesByChunkGroup.get(chunkGroup) as Array - ).forEach(staticQuery => { - allStaticQueries.add(staticQuery) - }) + if (chunkGroup) { + const staticQueriesForChunkGroup = + staticQueriesByChunkGroup.get(chunkGroup) + + if (staticQueriesForChunkGroup) { + staticQueriesForChunkGroup.forEach(staticQuery => { + allStaticQueries.add(staticQuery) + }) + } + + const slicesForChunkGroup = + pageSliceUsageByChunkGroup.get(chunkGroup) + + if (slicesForChunkGroup) { + allSlices = mergePreviouslyCollectedSlices( + slicesForChunkGroup, + allSlices + ) + } } } // modules, chunks, chunkgroups can all have not-deterministic orders so // just sort array of static queries we produced to ensure final result is deterministic const staticQueryHashes = Array.from(allStaticQueries).sort() + const slices = Object.keys(allSlices) + .sort() + .reduce((obj, key) => { + obj[key] = allSlices[key] + return obj + }, {}) + + const didStaticQueriesChange = !isEqual( + this.store + .getState() + .staticQueriesByTemplate.get(component.componentPath), + staticQueryHashes + ) - if ( - !isEqual( - this.store - .getState() - .staticQueriesByTemplate.get(component.componentPath), - staticQueryHashes - ) - ) { + const didSlicesChange = !isEqual( + this.store.getState().slicesByTemplate.get(component.componentPath), + slices + ) + + if (didStaticQueriesChange || didSlicesChange) { + if (component.isSlice) { + this.store.dispatch({ + type: `ADD_PENDING_SLICE_TEMPLATE_DATA_WRITE`, + payload: { + componentPath: component.componentPath, + sliceNames: component.pages, + }, + }) + } else { + this.store.dispatch({ + type: `ADD_PENDING_TEMPLATE_DATA_WRITE`, + payload: { + componentPath: component.componentPath, + pages: component.pages, + }, + }) + } + } + + if (didSlicesChange) { this.store.dispatch({ - type: `ADD_PENDING_TEMPLATE_DATA_WRITE`, + type: `SET_SLICES_BY_TEMPLATE`, payload: { componentPath: component.componentPath, - pages: component.pages, + slices, }, }) + } + if (didStaticQueriesChange) { this.store.dispatch({ type: `SET_STATIC_QUERIES_BY_TEMPLATE`, payload: { diff --git a/packages/gatsby/src/utils/websocket-manager.ts b/packages/gatsby/src/utils/websocket-manager.ts index 3e5ab2088b80e..3fb2240d98e22 100644 --- a/packages/gatsby/src/utils/websocket-manager.ts +++ b/packages/gatsby/src/utils/websocket-manager.ts @@ -13,7 +13,7 @@ import { findPageByPath } from "./find-page-by-path" import { Server as SocketIO, Socket } from "socket.io" import { getPageMode } from "./page-mode" -export interface IPageQueryResult { +export interface IPageOrSliceQueryResult { id: string result?: IPageDataWithQueryResult } @@ -23,7 +23,6 @@ export interface IStaticQueryResult { result: unknown // TODO: Improve this once we understand what the type is } -type PageResultsMap = Map type QueryResultsMap = Map function hashPaths(paths: Array): Array { @@ -39,7 +38,6 @@ export class WebsocketManager { activePaths: Set = new Set() clients: Set = new Set() errors: Map = new Map() - pageResults: PageResultsMap = new Map() staticQueryResults: QueryResultsMap = new Map() websocket: SocketIO | undefined @@ -184,9 +182,7 @@ export class WebsocketManager { } } - emitPageData = (data: IPageQueryResult): void => { - this.pageResults.set(data.id, data) - + emitPageData = (data: IPageOrSliceQueryResult): void => { if (this.websocket) { this.websocket.send({ type: `pageQueryResult`, payload: data }) @@ -205,6 +201,12 @@ export class WebsocketManager { } } + emitSliceData = (data: IPageOrSliceQueryResult): void => { + if (this.websocket) { + this.websocket.send({ type: `sliceQueryResult`, payload: data }) + } + } + emitError = (id: string, message?: string): void => { if (message) { this.errors.set(id, message) diff --git a/packages/gatsby/src/utils/worker/__tests__/queries.ts b/packages/gatsby/src/utils/worker/__tests__/queries.ts index 277b4d1597fee..42e07c00ead05 100644 --- a/packages/gatsby/src/utils/worker/__tests__/queries.ts +++ b/packages/gatsby/src/utils/worker/__tests__/queries.ts @@ -111,11 +111,13 @@ const pageQueryIds = [dummyPageFoo, dummyPageBar, ...dummyPages] const queryIdsSmall: IGroupedQueryIds = { pageQueryIds: [dummyPageFoo, dummyPageBar], staticQueryIds: [dummyStaticQuery.id], + sliceQueryIds: [], } const queryIdsBig: IGroupedQueryIds = { pageQueryIds, staticQueryIds: [dummyStaticQuery.id], + sliceQueryIds: [], } describeWhenLMDB(`worker (queries)`, () => { @@ -337,21 +339,25 @@ describeWhenLMDB(`worker (queries)`, () => { expect(spy).toHaveBeenNthCalledWith(1, { pageQueryIds: [], staticQueryIds: expect.toBeArrayOfSize(1), + sliceQueryIds: [], }) expect(spy).toHaveBeenNthCalledWith(2, { pageQueryIds: expect.toBeArrayOfSize(10), staticQueryIds: [], + sliceQueryIds: [], }) expect(spy).toHaveBeenNthCalledWith(3, { pageQueryIds: expect.toBeArrayOfSize(10), staticQueryIds: [], + sliceQueryIds: [], }) expect(spy).toHaveBeenNthCalledWith(4, { pageQueryIds: expect.toBeArrayOfSize(8), staticQueryIds: [], + sliceQueryIds: [], }) spy.mockRestore() @@ -363,7 +369,7 @@ describeWhenLMDB(`worker (queries)`, () => { const expectedActionShapes = { QUERY_START: [`componentPath`, `isPage`, `path`], - PAGE_QUERY_RUN: [`componentPath`, `isPage`, `path`, `resultHash`], + PAGE_QUERY_RUN: [`componentPath`, `queryType`, `path`, `resultHash`], ADD_PENDING_PAGE_DATA_WRITE: [`path`], } expect(result).toBeArrayOfSize(8) @@ -394,7 +400,7 @@ describeWhenLMDB(`worker (queries)`, () => { { payload: { componentPath: `/static-query-component.js`, - isPage: false, + queryType: `static`, path: `sq--q1`, queryHash: `q1-hash`, resultHash: `Dr5hgCDB+R0S9oRBWeZYj3lB7VI=`, @@ -426,7 +432,7 @@ describeWhenLMDB(`worker (queries)`, () => { { payload: { componentPath: `/foo.js`, - isPage: true, + queryType: `page`, path: `/foo`, resultHash: `8dW7PoqwZNk/0U8LO6kTj1qBCwU=`, }, @@ -441,7 +447,7 @@ describeWhenLMDB(`worker (queries)`, () => { { payload: { componentPath: `/bar.js`, - isPage: true, + queryType: `page`, path: `/bar`, resultHash: `iKmhf9XgbsfK7qJw0tw95pmGwJM=`, }, diff --git a/packages/gatsby/src/utils/worker/__tests__/share-state.ts b/packages/gatsby/src/utils/worker/__tests__/share-state.ts index 752654f5ce5a1..c74e1c3b32ea1 100644 --- a/packages/gatsby/src/utils/worker/__tests__/share-state.ts +++ b/packages/gatsby/src/utils/worker/__tests__/share-state.ts @@ -102,6 +102,7 @@ describe(`worker (share-state)`, () => { "componentPath": "/foo", "config": false, "isInBootstrap": true, + "isSlice": false, "pages": Set { "/foo/", }, @@ -124,6 +125,7 @@ describe(`worker (share-state)`, () => { "componentPath": "/foo", "config": false, "isInBootstrap": true, + "isSlice": false, "pages": Set { "/foo/", }, @@ -236,6 +238,7 @@ describe(`worker (share-state)`, () => { "componentPath": "/foo", "config": false, "isInBootstrap": true, + "isSlice": false, "pages": Object {}, "query": "I'm a page query", "serverData": false, diff --git a/packages/gatsby/src/utils/worker/child/index.ts b/packages/gatsby/src/utils/worker/child/index.ts index 13ac08e1d8038..f940ceb7fdebe 100644 --- a/packages/gatsby/src/utils/worker/child/index.ts +++ b/packages/gatsby/src/utils/worker/child/index.ts @@ -12,11 +12,11 @@ if (isWorker) { : {} } -// Note: this doesn't check for conflicts between module exports export { renderHTMLProd, renderHTMLDev, renderPartialHydrationProd, + renderSlices, } from "./render-html" export { setInferenceMetadata, buildSchema } from "./schema" export { setComponents, runQueries, saveQueriesDependencies } from "./queries" diff --git a/packages/gatsby/src/utils/worker/child/render-html.ts b/packages/gatsby/src/utils/worker/child/render-html.ts index 9636692fa23a9..2f960aba1374d 100644 --- a/packages/gatsby/src/utils/worker/child/render-html.ts +++ b/packages/gatsby/src/utils/worker/child/render-html.ts @@ -12,7 +12,11 @@ import { getScriptsAndStylesForTemplate, clearCache as clearAssetsMappingCache, } from "../../client-assets-for-template" -import { IPageDataWithQueryResult, readPageData } from "../../page-data" +import { + IPageDataWithQueryResult, + readPageData, + readSliceData, +} from "../../page-data" import type { IRenderHtmlResult } from "../../../commands/build-html" import { clearStaticQueryCaches, @@ -20,6 +24,8 @@ import { getStaticQueryContext, } from "../../static-query-utils" import { ServerLocation } from "@gatsbyjs/reach-router" +import { IGatsbySlice } from "../../../internal" +import { ensureFileContent } from "../../ensure-file-content" // we want to force posix-style joins, so Windows doesn't produce backslashes for urls const { join } = path.posix @@ -61,6 +67,23 @@ const inFlightResourcesForTemplate = new Map< Promise >() +const readStaticQueryContext = async ( + templatePath: string +): Promise> => { + const filePath = path.join( + // TODO: Better way to get this? + process.cwd(), + `.cache`, + `page-ssr`, + `sq-context`, + templatePath, + `sq-context.json` + ) + const rawSQContext = await fs.readFile(filePath, `utf-8`) + + return JSON.parse(rawSQContext) +} + function clearCaches(): void { clearStaticQueryCaches() resourcesForTemplateCache.clear() @@ -145,6 +168,7 @@ export const renderHTMLProd = async ({ const unsafeBuiltinsUsageByPagePath = {} const previewErrors = {} + const allSlicesProps = {} // Check if we need to do setup and cache clearing. Within same session we can reuse memoized data, // but it's not safe to reuse them in different sessions. Check description of `lastSessionId` for more details @@ -174,14 +198,19 @@ export const renderHTMLProd = async ({ const pageData = await readPageData(publicDir, pagePath) const resourcesForTemplate = await getResourcesForTemplate(pageData) - const { html, unsafeBuiltinsUsage } = + const { html, unsafeBuiltinsUsage, sliceData } = await htmlComponentRenderer.default({ pagePath, pageData, webpackCompilationHash, + context: { + isDuringBuild: true, + }, ...resourcesForTemplate, }) + allSlicesProps[pagePath] = sliceData + if (unsafeBuiltinsUsage.length > 0) { unsafeBuiltinsUsageByPagePath[pagePath] = unsafeBuiltinsUsage } @@ -229,7 +258,11 @@ export const renderHTMLProd = async ({ { concurrency: 2 } ) - return { unsafeBuiltinsUsageByPagePath, previewErrors } + return { + unsafeBuiltinsUsageByPagePath, + previewErrors, + slicesPropsPerPage: allSlicesProps, + } } // TODO: remove when DEV_SSR is done @@ -266,6 +299,9 @@ export const renderHTMLDev = async ({ try { const htmlString = await htmlComponentRenderer.default({ pagePath, + context: { + isDuringBuild: true, + }, }) return fs.outputFile(generateHtmlPath(outputDir, pagePath), htmlString) } catch (e) { @@ -413,3 +449,71 @@ export async function renderPartialHydrationProd({ }) } } + +export interface IRenderSliceResult { + chunks: 2 | 1 +} + +export interface IRenderSlicesResults { + [sliceName: string]: IRenderSliceResult +} + +export interface ISlicePropsEntry { + sliceId: string + sliceName: string + props: Record + hasChildren: boolean +} + +export async function renderSlices({ + slices, + htmlComponentRendererPath, + publicDir, + slicesProps, +}: { + publicDir: string + slices: Array<[string, IGatsbySlice]> + slicesProps: Array + htmlComponentRendererPath: string +}): Promise { + const htmlComponentRenderer = require(htmlComponentRendererPath) + + for (const { sliceId, props, sliceName, hasChildren } of slicesProps) { + const sliceEntry = slices.find(f => f[0] === sliceName) + if (!sliceEntry) { + throw new Error( + `Slice name "${sliceName}" not found when rendering slices` + ) + } + + const [_fileName, slice] = sliceEntry + + const staticQueryContext = await readStaticQueryContext( + slice.componentChunkName + ) + + const MAGIC_CHILDREN_STRING = `__DO_NOT_USE_OR_ELSE__` + const sliceData = await readSliceData(publicDir, slice.name) + + const html = await htmlComponentRenderer.renderSlice({ + slice, + staticQueryContext, + props: { + data: sliceData?.result?.data, + ...(hasChildren ? { children: MAGIC_CHILDREN_STRING } : {}), + ...props, + }, + }) + const split = html.split(MAGIC_CHILDREN_STRING) + + // TODO always generate both for now + let index = 1 + for (const htmlChunk of split) { + await ensureFileContent( + path.join(publicDir, `_gatsby`, `slices`, `${sliceId}-${index}.html`), + htmlChunk + ) + index++ + } + } +} diff --git a/packages/gatsby/src/utils/worker/pool.ts b/packages/gatsby/src/utils/worker/pool.ts index a27bcb41426ca..a32b3392b1916 100644 --- a/packages/gatsby/src/utils/worker/pool.ts +++ b/packages/gatsby/src/utils/worker/pool.ts @@ -73,7 +73,11 @@ export async function runQueriesInWorkersQueue( for (const segment of staticQuerySegments) { pool.single - .runQueries({ pageQueryIds: [], staticQueryIds: segment }) + .runQueries({ + pageQueryIds: [], + staticQueryIds: segment, + sliceQueryIds: [], + }) .then(replayWorkerActions) .then(() => { activity.tick(segment.length) @@ -83,7 +87,11 @@ export async function runQueriesInWorkersQueue( for (const segment of pageQuerySegments) { pool.single - .runQueries({ pageQueryIds: segment, staticQueryIds: [] }) + .runQueries({ + pageQueryIds: segment, + staticQueryIds: [], + sliceQueryIds: [], + }) .then(replayWorkerActions) .then(() => { activity.tick(segment.length) diff --git a/patches/v5/4-upgrade-reach-router.patch b/patches/v5/4-upgrade-reach-router.patch index 1a883108aef8f..35d9a255fe1e7 100644 --- a/patches/v5/4-upgrade-reach-router.patch +++ b/patches/v5/4-upgrade-reach-router.patch @@ -110,7 +110,7 @@ index 2c2ea7a3ea..d3457e0bff 100644 +import { match } from "@gatsbyjs/reach-router" import { joinPath } from "gatsby-core-utils" import { store, emitter } from "../redux/" - import { IGatsbyState, IGatsbyPage } from "../redux/types" + import { IGatsbyState, IGatsbyPage, IGatsbySlice } from "../redux/types" diff --git a/packages/gatsby/src/commands/serve.ts b/packages/gatsby/src/commands/serve.ts index 03d8a87321..b457cde7e7 100644 --- a/packages/gatsby/src/commands/serve.ts diff --git a/patches/v5/5-slices-api.patch b/patches/v5/5-slices-api.patch new file mode 100644 index 0000000000000..41f6cc10bb891 --- /dev/null +++ b/patches/v5/5-slices-api.patch @@ -0,0 +1,190 @@ +diff --git a/packages/gatsby/cache-dir/gatsby-browser-entry.js b/packages/gatsby/cache-dir/gatsby-browser-entry.js +index 9e0f1205bf..f0a7dcb841 100644 +--- a/packages/gatsby/cache-dir/gatsby-browser-entry.js ++++ b/packages/gatsby/cache-dir/gatsby-browser-entry.js +@@ -23,4 +23,5 @@ export { + + export { graphql, prefetchPathname } + export { StaticQueryContext, useStaticQuery } from "./static-query" ++export { Slice } from "./slice" + export * from "gatsby-script" +diff --git a/packages/gatsby/index.d.ts b/packages/gatsby/index.d.ts +index db2b6a78c1..f0e7745e01 100644 +--- a/packages/gatsby/index.d.ts ++++ b/packages/gatsby/index.d.ts +@@ -182,6 +182,30 @@ export type HeadFC = ( + props: HeadProps + ) => JSX.Element + ++type SerializableProps = ++ | ISerializableObject ++ | Array ++ | string ++ | number ++ | boolean ++ | null ++ | undefined ++ ++interface ISerializableObject { ++ [key: string]: SerializableProps ++} ++ ++export interface SliceProps { ++ alias: string ++ allowEmpty?: boolean ++ [key: string]: SerializableProps ++} ++ ++/** ++ * TODO ++ */ ++export const Slice = (props: SliceProps) => JSX.Element ++ + /** + * Props object passed into the [getServerData](https://www.gatsbyjs.com/docs/reference/rendering-options/server-side-rendering/) function. + */ +@@ -1207,6 +1231,14 @@ export interface Actions { + option?: ActionOptions + ): void + ++ /** @see https://www.gatsbyjs.com/docs/reference/config-files/actions/#createSlice */ ++ createSlice>( ++ this: void, ++ args: SliceInput, ++ plugin?: ActionPlugin, ++ option?: ActionOptions ++ ): void ++ + /** @see https://www.gatsbyjs.com/docs/actions/#deleteNode */ + deleteNode(node: NodeInput, plugin?: ActionPlugin): void + +@@ -1642,6 +1674,13 @@ export interface Page> { + context?: TContext + ownerNodeId?: string + defer?: boolean ++ slices?: Record ++} ++ ++export interface SliceInput> { ++ id: string ++ component: string ++ context?: TContext + } + + export interface IPluginRefObject { +diff --git a/packages/gatsby/src/redux/actions/restricted.ts b/packages/gatsby/src/redux/actions/restricted.ts +index cb105dfa6d..0ee5d3eda7 100644 +--- a/packages/gatsby/src/redux/actions/restricted.ts ++++ b/packages/gatsby/src/redux/actions/restricted.ts +@@ -1,4 +1,6 @@ +-import { camelCase } from "lodash" ++import camelCase from "lodash/camelCase" ++import isEqual from "lodash/isEqual" ++ + import { GraphQLSchema, GraphQLOutputType } from "graphql" + import { ActionCreator } from "redux" + import { ThunkAction } from "redux-thunk" +@@ -19,7 +21,12 @@ import { + IPrintTypeDefinitions, + ICreateResolverContext, + IGatsbyPluginContext, ++ ICreateSliceAction, + } from "../types" ++import { generateComponentChunkName } from "../../utils/js-chunk-names" ++import { store } from "../index" ++import normalizePath from "normalize-path" ++import { trackFeatureIsUsed } from "gatsby-telemetry" + + type RestrictionActionNames = + | "createFieldExtension" +@@ -27,6 +34,7 @@ type RestrictionActionNames = + | "createResolverContext" + | "addThirdPartySchema" + | "printTypeDefinitions" ++ | "createSlice" + + type SomeActionCreator = + | ActionCreator +@@ -420,6 +428,74 @@ export const actions = { + }) + } + }, ++ ++ createSlice: ( ++ payload: { ++ id: string ++ component: string ++ context: Record ++ }, ++ plugin: IGatsbyPlugin, ++ traceId?: string ++ ): ICreateSliceAction => { ++ if (_CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES) { ++ let name = `The plugin "${plugin.name}"` ++ if (plugin.name === `default-site-plugin`) { ++ name = `Your site's "gatsby-node.js"` ++ } ++ ++ if (!payload.id) { ++ const message = `${name} must set the page path when creating a slice` ++ report.panic({ ++ id: `11334`, ++ context: { ++ pluginName: name, ++ sliceObject: payload, ++ message, ++ }, ++ }) ++ } ++ if (!payload.component) { ++ report.panic({ ++ id: `11333`, ++ context: { ++ pluginName: name, ++ sliceObject: payload, ++ }, ++ }) ++ } ++ ++ trackFeatureIsUsed(`SliceAPI`) ++ const componentPath = normalizePath(payload.component) ++ ++ const oldSlice = store.getState().slices.get(payload.id) ++ const contextModified = ++ !!oldSlice && !isEqual(oldSlice.context, payload.context) ++ const componentModified = ++ !!oldSlice && !isEqual(oldSlice.componentPath, componentPath) ++ ++ return { ++ type: `CREATE_SLICE`, ++ plugin, ++ payload: { ++ componentChunkName: generateComponentChunkName( ++ payload.component, ++ `slice` ++ ), ++ componentPath, ++ // note: we use "name" internally instead of id ++ name: payload.id, ++ context: payload.context || {}, ++ updatedAt: Date.now(), ++ }, ++ traceId, ++ componentModified, ++ contextModified, ++ } ++ } else { ++ throw new Error(`createSlice is only available in Gatsby v5`) ++ } ++ }, + } + + const withDeprecationWarning = +@@ -540,4 +616,7 @@ export const availableActionsByAPI = mapAvailableActionsToAPIs({ + printTypeDefinitions: { + [ALLOWED_IN]: [`createSchemaCustomization`], + }, ++ createSlice: { ++ [ALLOWED_IN]: [`createPages`], ++ }, + }) diff --git a/patches/v5/5-slices-public-ts-types.patch b/patches/v5/5-slices-public-ts-types.patch deleted file mode 100644 index da7bdd05f487a..0000000000000 --- a/patches/v5/5-slices-public-ts-types.patch +++ /dev/null @@ -1,32 +0,0 @@ -diff --git a/packages/gatsby/index.d.ts b/packages/gatsby/index.d.ts -index 57c92929c1..dae8617ee5 100644 ---- a/packages/gatsby/index.d.ts -+++ b/packages/gatsby/index.d.ts -@@ -1227,6 +1227,14 @@ export interface Actions { - option?: ActionOptions - ): void - -+ /** @see https://www.gatsbyjs.com/docs/reference/config-files/actions/#createSlice */ -+ createSlice>( -+ this: void, -+ args: SliceInput, -+ plugin?: ActionPlugin, -+ option?: ActionOptions -+ ): void -+ - /** @see https://www.gatsbyjs.com/docs/actions/#deleteNode */ - deleteNode(node: NodeInput, plugin?: ActionPlugin): void - -@@ -1664,6 +1672,12 @@ export interface Page> { - defer?: boolean - } - -+export interface SliceInput> { -+ id: string -+ component: string -+ context?: TContext -+} -+ - export interface IPluginRefObject { - resolve: string - options?: IPluginRefOptions