perf(gatsby): Enable fast filters for $nin comparator (#24184)

packages/gatsby/src/redux/nodes.ts

@@ -4,7 +4,7 @@ import { createPageDependency } from "./actions/add-page-dependency"
 import { IDbQueryElemMatch } from "../db/common/query"
 
 // Only list supported ops here. "CacheableFilterOp"
-type FilterOp = "$eq" | "$ne" | "$lt" | "$lte" | "$gt" | "$gte" | "$in"
+type FilterOp = "$eq" | "$ne" | "$lt" | "$lte" | "$gt" | "$gte" | "$in" | "$nin"
 // Note: `undefined` is an encoding for a property that does not exist
 type FilterValueNullable =
   | string
@@ -188,20 +188,23 @@ export function postIndexingMetaSetup(
   filterCache: IFilterCache,
   op: FilterOp
 ): void {
-  if (op === `$ne`) {
-    postIndexingMetaSetupNe(filterCache)
+  if (op === `$ne` || op === `$nin`) {
+    postIndexingMetaSetupNeNin(filterCache)
   } else if ([`$lt`, `$lte`, `$gt`, `$gte`].includes(op)) {
     postIndexingMetaSetupLtLteGtGte(filterCache, op)
   }
 }
 
-function postIndexingMetaSetupNe(filterCache: IFilterCache): void {
+function postIndexingMetaSetupNeNin(filterCache: IFilterCache): void {
   // Note: edge cases regarding `null` and `undefined`. Here `undefined` signals
   // that the property did not exist as sift does not support actual `undefined`
-  // values. For $ne, `null` only returns nodes that actually have the property
+  // values.
+  // For $ne, `null` only returns nodes that actually have the property
   // and in that case the property cannot be `null` either. For any other value,
   // $ne will return all nodes where the value is not actually the needle,
   // including nodes where the value is null.
+  // A $nin does the same as an $ne except it filters multiple values instead
+  // of just one.
 
   // For `$ne` we will take the list of all targeted nodes and eliminate the
   // bucket of nodes with a particular value, if it exists at all. So for that
@@ -643,6 +646,37 @@ export const getNodesFromCacheByValue = (
     return set
   }
 
+  if (op === `$nin`) {
+    // This is essentially the same as the $ne operator, just with multiple
+    // values to exclude.
+
+    if (!Array.isArray(filterValue)) {
+      throw new Error(`The $nin operator expects an array as value`)
+    }
+
+    const values: Set<FilterValueNullable> = new Set(filterValue)
+    const set = new Set(filterCache.meta.nodesUnordered)
+
+    // Do the action for "$ne" for each element in the set of values
+    values.forEach(filterValue => {
+      if (filterValue === null) {
+        // Edge case: $nin with `null` returns only the nodes that contain the
+        // full path and that don't resolve to null, so drop `undefined` as well
+        let cache = filterCache.byValue.get(undefined)
+        if (cache) cache.forEach(node => set.delete(node))
+        cache = filterCache.byValue.get(null)
+        if (cache) cache.forEach(node => set.delete(node))
+      } else {
+        // Not excluding null so it should include undefined leafs or leafs
+        // where only the partial path exists for whatever reason.
+        const cache = filterCache.byValue.get(filterValue)
+        if (cache) cache.forEach(node => set.delete(node))
+      }
+    })
+
+    return set
+  }
+
   if (op === `$ne`) {
     const set = new Set(filterCache.meta.nodesUnordered)
 

packages/gatsby/src/redux/run-sift.js

@@ -19,7 +19,7 @@ const {
   getNode: siftGetNode,
 } = require(`./nodes`)
 
-const FAST_OPS = [`$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`]
+const FAST_OPS = [`$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`, `$nin`]
 
 // More of a testing mechanic, to verify whether last runSift call used Sift
 let lastFilterUsedSift = false

packages/gatsby/src/schema/__tests__/run-query.js

@@ -1608,7 +1608,7 @@ it(`should use the cache argument`, async () => {
 
       describe(`$nin`, () => {
         it(`handles the nin operator for array [5]`, async () => {
-          const [result] = await runSlowFilter({ anArray: { nin: [5] } })
+          const [result] = await runFastFilter({ anArray: { nin: [5] } })
 
           // Since the array does not contain `null`, the query should also return the
           // nodes that do not have the field at all (NULL).
@@ -1625,7 +1625,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for array [null]`, async () => {
-          const [result] = await runSlowFilter({
+          const [result] = await runFastFilter({
             nullArray: { nin: [null] },
           })
 
@@ -1637,7 +1637,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for strings`, async () => {
-          const [result] = await runSlowFilter({
+          const [result] = await runFastFilter({
             string: { nin: [`b`, `c`] },
           })
 
@@ -1649,7 +1649,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for ints`, async () => {
-          const [result] = await runSlowFilter({ index: { nin: [0, 2] } })
+          const [result] = await runFastFilter({ index: { nin: [0, 2] } })
 
           expect(result.length).toEqual(1)
           result.forEach(node => {
@@ -1659,7 +1659,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for floats`, async () => {
-          const [result] = await runSlowFilter({ float: { nin: [1.5] } })
+          const [result] = await runFastFilter({ float: { nin: [1.5] } })
 
           expect(result.length).toEqual(2)
           result.forEach(node => {
@@ -1668,8 +1668,8 @@ it(`should use the cache argument`, async () => {
           })
         })
 
-        it(`handles the nin operator for booleans`, async () => {
-          const [result] = await runSlowFilter({
+        it(`handles the nin operator for boolean and null on boolean`, async () => {
+          const [result] = await runFastFilter({
             boolean: { nin: [true, null] },
           })
 
@@ -1683,7 +1683,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for double null`, async () => {
-          const [result] = await runSlowFilter({
+          const [result] = await runFastFilter({
             nil: { nin: [null, null] },
           })
 
@@ -1697,7 +1697,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for null in int+null`, async () => {
-          const [result] = await runSlowFilter({
+          const [result] = await runFastFilter({
             nil: { nin: [5, null] },
           })
 
@@ -1711,7 +1711,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the nin operator for int in int+null`, async () => {
-          const [result] = await runSlowFilter({
+          const [result] = await runFastFilter({
             index: { nin: [2, null] },
           })