feat(gatsby): Add nodes db module (#9416)

In preparation for #9338, I've added a new nodes db module that can be implemented by an appropriate db nodes backend. Which is only redux as of now. The loki PR will eventually be implemented so that it can be featured flagged on/off using this functionality. I also added `getNodesByType` since it makes sense even without loki.
gatsbyjs · Oct 29, 2018 · 7d31fe7 · 7d31fe7
1 parent 343f9e6
commit 7d31fe7
Show file tree

Hide file tree

Showing 32 changed files with 284 additions and 208 deletions.
diff --git a/packages/gatsby-source-contentful/package.json b/packages/gatsby-source-contentful/package.json
@@ -34,7 +34,7 @@
   ],
   "license": "MIT",
   "peerDependencies": {
-    "gatsby": ">2.0.0-alpha"
+    "gatsby": "^2.0.33"
   },
   "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-contentful",
   "scripts": {

diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js
@@ -216,17 +216,15 @@ exports.sourceNodes = async (
 // Check if there are any ContentfulAsset nodes and if gatsby-image is installed. If so,
 // add fragments for ContentfulAsset and gatsby-image. The fragment will cause an error
 // if there's not ContentfulAsset nodes and without gatsby-image, the fragment is useless.
-exports.onPreExtractQueries = async ({ store, getNodes }) => {
+exports.onPreExtractQueries = async ({ store, getNodesByType }) => {
   const program = store.getState().program
 
   const CACHE_DIR = path.resolve(
     `${program.directory}/.cache/contentful/assets/`
   )
   await fs.ensureDir(CACHE_DIR)
 
-  const nodes = getNodes()
-
-  if (!nodes.some(n => n.internal.type === `ContentfulAsset`)) {
+  if (getNodesByType(`ContentfulAsset`).length == 0) {
     return
   }
 

diff --git a/packages/gatsby-transformer-remark/package.json b/packages/gatsby-transformer-remark/package.json
@@ -42,7 +42,7 @@
   ],
   "license": "MIT",
   "peerDependencies": {
-    "gatsby": ">2.0.0-alpha"
+    "gatsby": "^2.0.33"
   },
   "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-remark",
   "scripts": {

diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js
@@ -11,7 +11,12 @@ const {
 const extendNodeType = require(`../extend-node-type`)
 
 // given a set of nodes and a query, return the result of the query
-async function queryResult(nodes, fragment, { types = [] } = {}, additionalParameters) {
+async function queryResult(
+  nodes,
+  fragment,
+  { types = [] } = {},
+  additionalParameters
+) {
   const inferredFields = inferObjectStructureFromNodes({
     nodes,
     types: [...types],
@@ -23,7 +28,7 @@ async function queryResult(nodes, fragment, { types = [] } = {}, additionalParam
         get: () => null,
         set: () => null,
       },
-      getNodes: () => [],
+      getNodesByType: type => [],
       ...additionalParameters,
     },
     {
@@ -70,7 +75,13 @@ async function queryResult(nodes, fragment, { types = [] } = {}, additionalParam
   return result
 }
 
-const bootstrapTest = (label, content, query, test, additionalParameters = {}) => {
+const bootstrapTest = (
+  label,
+  content,
+  query,
+  test,
+  additionalParameters = {}
+) => {
   const node = {
     id: `whatever`,
     children: [],
@@ -82,7 +93,7 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) =
   // Make some fake functions its expecting.
   const loadNodeContent = node => Promise.resolve(node.content)
 
-  it(label, async (done) => {
+  it(label, async done => {
     node.content = content
     const createNode = markdownNode => {
       queryResult(
@@ -96,8 +107,7 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) =
         try {
           test(result.data.listNode[0])
           done()
-        }
-        catch(err) {
+        } catch (err) {
           done.fail(err)
         }
       })
@@ -106,19 +116,19 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) =
     const actions = { createNode, createParentChildLink }
     const createNodeId = jest.fn()
     createNodeId.mockReturnValue(`uuid-from-gatsby`)
-    await onCreateNode({
-      node,
-      loadNodeContent,
-      actions,
-      createNodeId,
-    },
-    { ...additionalParameters }
+    await onCreateNode(
+      {
+        node,
+        loadNodeContent,
+        actions,
+        createNodeId,
+      },
+      { ...additionalParameters }
     )
-    })
+  })
 }
 
 describe(`Excerpt is generated correctly from schema`, () => {
-
   bootstrapTest(
     `correctly loads an excerpt`,
     `---
@@ -131,7 +141,7 @@ Where oh where is my little pony?`,
         title
     }
     `,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.excerpt).toMatch(`Where oh where is my little pony?`)
     }
@@ -148,7 +158,7 @@ date: "2017-09-18T23:19:51.246Z"
         title
     }
     `,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.excerpt).toMatch(``)
     }
@@ -171,7 +181,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
         title
     }
     `,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.excerpt).toMatch(`Where oh where is my little pony?`)
     },
@@ -194,7 +204,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
         title
     }
     `,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.excerpt.length).toBe(139)
     }
@@ -208,7 +218,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
         title
     }
     `,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.excerpt.length).toBe(46)
     }
@@ -222,7 +232,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
         title
     }
     `,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.excerpt.length).toBe(50)
     }
@@ -248,16 +258,15 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
     frontmatter {
         title
     }`,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
-      expect(node.wordCount).toEqual(
-        {
+      expect(node.wordCount).toEqual({
         paragraphs: 2,
         sentences: 19,
         words: 150,
-        }
-      )
-    })
+      })
+    }
+  )
 
   const content = `---
 title: "my little pony"
@@ -276,16 +285,15 @@ date: "2017-09-18T23:19:51.246Z"
     frontmatter {
         title
     }`,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
-      expect(node.wordCount).toEqual(
-        {
+      expect(node.wordCount).toEqual({
         paragraphs: null,
         sentences: null,
         words: null,
-        }
-      )
-    })
+      })
+    }
+  )
 
   bootstrapTest(
     `correctly uses a default value for timeToRead`,
@@ -294,10 +302,11 @@ date: "2017-09-18T23:19:51.246Z"
     frontmatter {
         title
     }`,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.timeToRead).toBe(1)
-    })
+    }
+  )
 })
 
 describe(`Table of contents is generated correctly from schema`, () => {
@@ -322,11 +331,12 @@ some other text
     frontmatter {
         title
     }`,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(console.warn).toBeCalled()
       expect(node.tableOfContents).toBe(null)
-    })
+    }
+  )
 
   bootstrapTest(
     `correctly generates table of contents`,
@@ -350,9 +360,10 @@ final text
     frontmatter {
         title
     }`,
-  (node) => {
-    expect(node).toMatchSnapshot()
-  })
+    node => {
+      expect(node).toMatchSnapshot()
+    }
+  )
 })
 
 describe(`Links are correctly prefixed`, () => {
@@ -366,7 +377,7 @@ This is [a reference]
 [a reference]: /path/to/page2
 `,
     `html`,
-    (node) => {
+    node => {
       expect(node).toMatchSnapshot()
       expect(node.html).toMatch(`<a href="/prefix/path/to/page1">`)
       expect(node.html).toMatch(`<a href="/prefix/path/to/page2">`)

diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js
@@ -62,7 +62,7 @@ const withPathPrefix = (url, pathPrefix) =>
 const ASTPromiseMap = new Map()
 
 module.exports = (
-  { type, store, pathPrefix, getNode, getNodes, cache, reporter },
+  { type, store, pathPrefix, getNode, getNodesByType, cache, reporter },
   pluginOptions
 ) => {
   if (type.name !== `MarkdownRemark`) {
@@ -74,7 +74,13 @@ module.exports = (
 
   return new Promise((resolve, reject) => {
     // Setup Remark.
-    const { commonmark = true, footnotes = true, pedantic = true, gfm = true, blocks } = pluginOptions
+    const {
+      commonmark = true,
+      footnotes = true,
+      pedantic = true,
+      gfm = true,
+      blocks,
+    } = pluginOptions
     const remarkOptions = {
       gfm,
       commonmark,
@@ -113,7 +119,7 @@ module.exports = (
       } else {
         const ASTGenerationPromise = new Promise(async resolve => {
           if (process.env.NODE_ENV !== `production` || !fileNodes) {
-            fileNodes = getNodes().filter(n => n.internal.type === `File`)
+            fileNodes = getNodesByType(`File`)
           }
           const ast = await new Promise((resolve, reject) => {
             // Use Bluebird's Promise function "each" to run remark plugins serially.
@@ -180,7 +186,7 @@ module.exports = (
               // typegen plugins just modify the auto-generated types to add derived fields
               // as well as computationally expensive fields.
               if (process.env.NODE_ENV !== `production` || !fileNodes) {
-                fileNodes = getNodes().filter(n => n.internal.type === `File`)
+                fileNodes = getNodesByType(`File`)
               }
               // Use Bluebird's Promise function "each" to run remark plugins serially.
               Promise.each(pluginOptions.plugins, plugin => {
@@ -249,10 +255,16 @@ module.exports = (
           const addSlugToUrl = function(node) {
             if (node.url) {
               if (_.get(markdownNode, pathToSlugField) === undefined) {
-                console.warn(`Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`)
+                console.warn(
+                  `Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`
+                )
                 return null
               }
-              node.url = [pathPrefix, _.get(markdownNode, pathToSlugField), node.url]
+              node.url = [
+                pathPrefix,
+                _.get(markdownNode, pathToSlugField),
+                node.url,
+              ]
                 .join(`/`)
                 .replace(/\/\//g, `/`)
             }

diff --git a/packages/gatsby-transformer-screenshot/package.json b/packages/gatsby-transformer-screenshot/package.json
@@ -25,7 +25,7 @@
   "license": "MIT",
   "main": "index.js",
   "peerDependencies": {
-    "gatsby": ">2.0.15"
+    "gatsby": "^2.0.33"
   },
   "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-screenshot",
   "scripts": {

diff --git a/packages/gatsby-transformer-screenshot/src/gatsby-node.js b/packages/gatsby-transformer-screenshot/src/gatsby-node.js
@@ -16,13 +16,11 @@ const screenshotQueue = new Queue(
 )
 
 exports.onPreBootstrap = (
-  { store, cache, actions, createNodeId, getNodes, createContentDigest },
+  { store, cache, actions, createNodeId, getNodesByType, createContentDigest },
   pluginOptions
 ) => {
   const { createNode, touchNode } = actions
-  const screenshotNodes = getNodes().filter(
-    n => n.internal.type === `Screenshot`
-  )
+  const screenshotNodes = getNodesByType(`Screenshot`)
 
   if (screenshotNodes.length === 0) {
     return null

diff --git a/packages/gatsby-transformer-sharp/package.json b/packages/gatsby-transformer-sharp/package.json
@@ -29,7 +29,7 @@
   ],
   "license": "MIT",
   "peerDependencies": {
-    "gatsby": ">2.0.0-alpha",
+    "gatsby": "^2.0.33",
     "gatsby-plugin-sharp": "^2.0.0-beta.3"
   },
   "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-sharp",

diff --git a/packages/gatsby-transformer-sharp/src/gatsby-node.js b/packages/gatsby-transformer-sharp/src/gatsby-node.js
@@ -3,14 +3,12 @@ const fs = require(`fs-extra`)
 exports.onCreateNode = require(`./on-node-create`)
 exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`)
 
-exports.onPreExtractQueries = async ({ store, getNodes }) => {
+exports.onPreExtractQueries = async ({ store, getNodesByType }) => {
   const program = store.getState().program
 
   // Check if there are any ImageSharp nodes. If so add fragments for ImageSharp.
   // The fragment will cause an error if there are no ImageSharp nodes.
-  const nodes = getNodes()
-
-  if (!nodes.some(n => n.internal.type === `ImageSharp`)) {
+  if (getNodesByType(`ImageSharp`).length == 0) {
     return
   }
 

diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js
@@ -52,6 +52,10 @@ module.exports = async (args: BootstrapArgs) => {
   const spanArgs = args.parentSpan ? { childOf: args.parentSpan } : {}
   const bootstrapSpan = tracer.startSpan(`bootstrap`, spanArgs)
 
+  // Start plugin runner which listens to the store
+  // and invokes Gatsby API based on actions.
+  require(`../redux/plugin-runner`)
+
   const program = {
     ...args,
     // Fix program directory path for windows env.