feat(gatsby): Materialization (#16091)

Add a node backend independent materialization layer that triggers when nodes are searched on fields only available through resolvers.

In addition, this fixes various incompatibilities that loki had due to never being tested through materialization.

In addition, node tracking is now happening on node-model level too.

In addition, this adds extensions to decide which fields are sortable and searchable and which needs resolve, so we can materialize.

Author: freiksenet

packages/gatsby/src/bootstrap/__tests__/graphql-runner.js

@@ -8,7 +8,9 @@ const createStore = (schema = {}) => {
     getState: () => {
       return {
         schema,
-        schemaCustomization: {},
+        schemaCustomization: {
+          composer: {},
+        },
       }
     },
   }

packages/gatsby/src/bootstrap/graphql-runner.js

@@ -1,53 +1,45 @@
-const { graphql } = require(`graphql`)
 const stackTrace = require(`stack-trace`)
-const withResolverContext = require(`../schema/context`)
-const errorParser = require(`../query/error-parser`).default
 
-const createGraphqlRunner = (store, reporter) => (query, context = {}) => {
-  const { schema, schemaCustomization } = store.getState()
+const GraphQLRunner = require(`../query/graphql-runner`)
+const errorParser = require(`../query/error-parser`).default
 
-  return graphql(
-    schema,
-    query,
-    context,
-    withResolverContext(context, schema, schemaCustomization.context),
-    context
-  ).then(result => {
-    if (result.errors) {
-      const structuredErrors = result.errors
-        .map(e => {
-          // Find the file where graphql was called.
-          const file = stackTrace
-            .parse(e)
-            .find(file => /createPages/.test(file.functionName))
+module.exports = (store, reporter) => {
+  const runner = new GraphQLRunner(store)
+  return (query, context) =>
+    runner.query(query, context).then(result => {
+      if (result.errors) {
+        const structuredErrors = result.errors
+          .map(e => {
+            // Find the file where graphql was called.
+            const file = stackTrace
+              .parse(e)
+              .find(file => /createPages/.test(file.functionName))
 
-          if (file) {
-            const structuredError = errorParser({
-              message: e.message,
-              location: {
-                start: { line: file.lineNumber, column: file.columnNumber },
-              },
-              filePath: file.fileName,
-            })
-            structuredError.context = {
-              ...structuredError.context,
-              fromGraphQLFunction: true,
+            if (file) {
+              const structuredError = errorParser({
+                message: e.message,
+                location: {
+                  start: { line: file.lineNumber, column: file.columnNumber },
+                },
+                filePath: file.fileName,
+              })
+              structuredError.context = {
+                ...structuredError.context,
+                fromGraphQLFunction: true,
+              }
+              return structuredError
             }
-            return structuredError
-          }
 
-          return null
-        })
-        .filter(Boolean)
+            return null
+          })
+          .filter(Boolean)
 
-      if (structuredErrors.length) {
-        // panic on build exits the process
-        reporter.panicOnBuild(structuredErrors)
+        if (structuredErrors.length) {
+          // panic on build exits the process
+          reporter.panicOnBuild(structuredErrors)
+        }
       }
-    }
 
-    return result
-  })
+      return result
+    })
 }
-
-module.exports = createGraphqlRunner

packages/gatsby/src/bootstrap/index.js

@@ -20,7 +20,6 @@ const report = require(`gatsby-cli/lib/reporter`)
 const getConfigFile = require(`./get-config-file`)
 const tracer = require(`opentracing`).globalTracer()
 const preferDefault = require(`./prefer-default`)
-const nodeTracking = require(`../db/node-tracking`)
 // Add `util.promisify` polyfill for old node versions
 require(`util.promisify/shim`)()
 
@@ -247,11 +246,6 @@ module.exports = async (args: BootstrapArgs) => {
     activity.end()
   }
 
-  // By now, our nodes database has been loaded, so ensure that we
-  // have tracked all inline objects
-  nodeTracking.trackDbNodes()
-
-  // Copy our site files to the root of the site.
   activity = report.activityTimer(`copy gatsby files`, {
     parentSpan: bootstrapSpan,
   })

packages/gatsby/src/commands/develop.js

@@ -39,7 +39,6 @@ const db = require(`../db`)
 const detectPortInUseAndPrompt = require(`../utils/detect-port-in-use-and-prompt`)
 const onExit = require(`signal-exit`)
 const queryUtil = require(`../query`)
-const queryQueue = require(`../query/queue`)
 const queryWatcher = require(`../query/query-watcher`)
 const requiresWriter = require(`../bootstrap/requires-writer`)
 
@@ -147,10 +146,16 @@ async function startServer(program, { activity }) {
     graphqlEndpoint,
     graphqlHTTP(() => {
       const { schema, schemaCustomization } = store.getState()
+
       return {
         schema,
         graphiql: false,
-        context: withResolverContext({}, schema, schemaCustomization.context),
+        context: withResolverContext({
+          schema,
+          schemaComposer: schemaCustomization.composer,
+          context: {},
+          customContext: schemaCustomization.context,
+        }),
         customFormatErrorFn(err) {
           return {
             ...formatError(err),
@@ -368,7 +373,7 @@ module.exports = async (program: any) => {
   await waitJobsFinished()
   requiresWriter.startListener()
   db.startAutosave()
-  queryUtil.startListening(queryQueue.createDevelopQueue())
+  queryUtil.startListeningToDevelopQueue()
   queryWatcher.startWatchDeletePage()
 
   activity = report.activityTimer(`start webpack server`)

packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap renamed to packages/gatsby/src/db/__tests__/__snapshots__/sanitize-node.js.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`track root nodes Node sanitization Remove not supported fields / values 1`] = `
+exports[`node sanitization Remove not supported fields / values 1`] = `
 Object {
   "children": Array [],
   "id": "id1",