perf(db-mongodb): improve performance of all operations, up to 50% faster (#9594)

This PR improves speed and memory efficiency across all operations with
the Mongoose adapter.

### How?

- Removes Mongoose layer from all database calls, instead uses MongoDB
directly. (this doesn't remove building mongoose schema since it's still
needed for indexes + users in theory can use it)
- Replaces deep copying of read results using
`JSON.parse(JSON.stringify(data))` with the `transform` `operation:
'read'` function which converts Date's, ObjectID's in relationships /
joins to strings. As before, it also handles transformations for write
operations.
- Faster `hasNearConstraint` for potentially large `where`'s
- `traverseFields` now can accept `flattenedFields` which we use in
`transform`. Less recursive calls with tabs/rows/collapsible

Additional fixes
- Uses current transaction for querying nested relationships properties
in `buildQuery`, previously it wasn't used which could've led to wrong
results
- Allows to clear not required point fields with passing `null` from the
Local API. Previously it didn't work in both, MongoDB and Postgres

Benchmarks using this file
https://github.com/payloadcms/payload/blob/chore/db-benchmark/test/_community/int.spec.ts

### Small Dataset Performance

| Metric | Before Optimization | After Optimization | Improvement (%) |

|---------------------------|---------------------|--------------------|-----------------|
| Average FULL (ms) | 1170 | 844 | 27.86% |
| `payload.db.create` (ms) | 1413 | 691 | 51.12% |
| `payload.db.find` (ms) | 2856 | 2204 | 22.83% |
| `payload.db.deleteMany` (ms) | 15206 | 8439 | 44.53% |
| `payload.db.updateOne` (ms) | 21444 | 12162 | 43.30% |
| `payload.db.findOne` (ms) | 159 | 112 | 29.56% |
| `payload.db.deleteOne` (ms) | 3729 | 2578 | 30.89% |
| DB small FULL (ms) | 64473 | 46451 | 27.93% |

---

### Medium Dataset Performance

| Metric | Before Optimization | After Optimization | Improvement (%) |

|---------------------------|---------------------|--------------------|-----------------|
| Average FULL (ms) | 9407 | 6210 | 33.99% |
| `payload.db.create` (ms) | 10270 | 4321 | 57.93% |
| `payload.db.find` (ms) | 20814 | 16036 | 22.93% |
| `payload.db.deleteMany` (ms) | 126351 | 61789 | 51.11% |
| `payload.db.updateOne` (ms) | 201782 | 99943 | 50.49% |
| `payload.db.findOne` (ms) | 1081 | 817 | 24.43% |
| `payload.db.deleteOne` (ms) | 28534 | 23363 | 18.12% |
| DB medium FULL (ms) | 519518 | 342194 | 34.13% |

---

### Large Dataset Performance

| Metric | Before Optimization | After Optimization | Improvement (%) |

|---------------------------|---------------------|--------------------|-----------------|
| Average FULL (ms) | 26575 | 17509 | 34.14% |
| `payload.db.create` (ms) | 29085 | 12196 | 58.08% |
| `payload.db.find` (ms) | 58497 | 43838 | 25.04% |
| `payload.db.deleteMany` (ms) | 372195 | 173218 | 53.47% |
| `payload.db.updateOne` (ms) | 544089 | 288350 | 47.00% |
| `payload.db.findOne` (ms) | 3058 | 2197 | 28.14% |
| `payload.db.deleteOne` (ms) | 82444 | 64730 | 21.49% |
| DB large FULL (ms) | 1461097 | 969714 | 33.62% |

Author: r1tsuu

packages/db-mongodb/src/count.ts

@@ -1,52 +1,47 @@
 import type { CountOptions } from 'mongodb'
 import type { Count } from 'payload'
 
-import { flattenWhereToOperators } from 'payload'
-
 import type { MongooseAdapter } from './index.js'
 
+import { getHasNearConstraint } from './utilities/getHasNearConstraint.js'
 import { getSession } from './utilities/getSession.js'
 
 export const count: Count = async function count(
   this: MongooseAdapter,
   { collection, locale, req, where },
 ) {
   const Model = this.collections[collection]
-  const options: CountOptions = {
-    session: await getSession(this, req),
-  }
+  const session = await getSession(this, req)
 
-  let hasNearConstraint = false
-
-  if (where) {
-    const constraints = flattenWhereToOperators(where)
-    hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near'))
-  }
+  const hasNearConstraint = getHasNearConstraint(where)
 
   const query = await Model.buildQuery({
     locale,
     payload: this.payload,
+    session,
     where,
   })
 
   // useEstimatedCount is faster, but not accurate, as it ignores any filters. It is thus set to true if there are no filters.
   const useEstimatedCount = hasNearConstraint || !query || Object.keys(query).length === 0
 
-  if (!useEstimatedCount && Object.keys(query).length === 0 && this.disableIndexHints !== true) {
-    // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
-    // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
-    // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
-    // the correct indexed field
-    options.hint = {
-      _id: 1,
-    }
-  }
-
   let result: number
   if (useEstimatedCount) {
-    result = await Model.estimatedDocumentCount({ session: options.session })
+    result = await Model.collection.estimatedDocumentCount()
   } else {
-    result = await Model.countDocuments(query, options)
+    const options: CountOptions = { session }
+
+    if (this.disableIndexHints !== true) {
+      // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
+      // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
+      // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
+      // the correct indexed field
+      options.hint = {
+        _id: 1,
+      }
+    }
+
+    result = await Model.collection.countDocuments(query, options)
   }
 
   return {

packages/db-mongodb/src/countGlobalVersions.ts

@@ -1,52 +1,47 @@
 import type { CountOptions } from 'mongodb'
 import type { CountGlobalVersions } from 'payload'
 
-import { flattenWhereToOperators } from 'payload'
-
 import type { MongooseAdapter } from './index.js'
 
+import { getHasNearConstraint } from './utilities/getHasNearConstraint.js'
 import { getSession } from './utilities/getSession.js'
 
 export const countGlobalVersions: CountGlobalVersions = async function countGlobalVersions(
   this: MongooseAdapter,
   { global, locale, req, where },
 ) {
   const Model = this.versions[global]
-  const options: CountOptions = {
-    session: await getSession(this, req),
-  }
+  const session = await getSession(this, req)
 
-  let hasNearConstraint = false
-
-  if (where) {
-    const constraints = flattenWhereToOperators(where)
-    hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near'))
-  }
+  const hasNearConstraint = getHasNearConstraint(where)
 
   const query = await Model.buildQuery({
     locale,
     payload: this.payload,
+    session,
     where,
   })
 
   // useEstimatedCount is faster, but not accurate, as it ignores any filters. It is thus set to true if there are no filters.
   const useEstimatedCount = hasNearConstraint || !query || Object.keys(query).length === 0
 
-  if (!useEstimatedCount && Object.keys(query).length === 0 && this.disableIndexHints !== true) {
-    // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
-    // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
-    // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
-    // the correct indexed field
-    options.hint = {
-      _id: 1,
-    }
-  }
-
   let result: number
   if (useEstimatedCount) {
-    result = await Model.estimatedDocumentCount({ session: options.session })
+    result = await Model.collection.estimatedDocumentCount()
   } else {
-    result = await Model.countDocuments(query, options)
+    const options: CountOptions = { session }
+
+    if (Object.keys(query).length === 0 && this.disableIndexHints !== true) {
+      // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
+      // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
+      // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
+      // the correct indexed field
+      options.hint = {
+        _id: 1,
+      }
+    }
+
+    result = await Model.collection.countDocuments(query, options)
   }
 
   return {

packages/db-mongodb/src/countVersions.ts

@@ -1,52 +1,47 @@
 import type { CountOptions } from 'mongodb'
 import type { CountVersions } from 'payload'
 
-import { flattenWhereToOperators } from 'payload'
-
 import type { MongooseAdapter } from './index.js'
 
+import { getHasNearConstraint } from './utilities/getHasNearConstraint.js'
 import { getSession } from './utilities/getSession.js'
 
 export const countVersions: CountVersions = async function countVersions(
   this: MongooseAdapter,
   { collection, locale, req, where },
 ) {
   const Model = this.versions[collection]
-  const options: CountOptions = {
-    session: await getSession(this, req),
-  }
+  const session = await getSession(this, req)
 
-  let hasNearConstraint = false
-
-  if (where) {
-    const constraints = flattenWhereToOperators(where)
-    hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near'))
-  }
+  const hasNearConstraint = getHasNearConstraint(where)
 
   const query = await Model.buildQuery({
     locale,
     payload: this.payload,
+    session,
     where,
   })
 
   // useEstimatedCount is faster, but not accurate, as it ignores any filters. It is thus set to true if there are no filters.
   const useEstimatedCount = hasNearConstraint || !query || Object.keys(query).length === 0
 
-  if (!useEstimatedCount && Object.keys(query).length === 0 && this.disableIndexHints !== true) {
-    // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
-    // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
-    // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
-    // the correct indexed field
-    options.hint = {
-      _id: 1,
-    }
-  }
-
   let result: number
   if (useEstimatedCount) {
-    result = await Model.estimatedDocumentCount({ session: options.session })
+    result = await Model.collection.estimatedDocumentCount()
   } else {
-    result = await Model.countDocuments(query, options)
+    const options: CountOptions = { session }
+
+    if (this.disableIndexHints !== true) {
+      // Improve the performance of the countDocuments query which is used if useEstimatedCount is set to false by adding
+      // a hint. By default, if no hint is provided, MongoDB does not use an indexed field to count the returned documents,
+      // which makes queries very slow. This only happens when no query (filter) is provided. If one is provided, it uses
+      // the correct indexed field
+      options.hint = {
+        _id: 1,
+      }
+    }
+
+    result = await Model.collection.countDocuments(query, options)
   }
 
   return {

packages/db-mongodb/src/create.ts

@@ -1,48 +1,44 @@
-import type { CreateOptions } from 'mongoose'
-import type { Create, Document } from 'payload'
+import type { Create } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
 import { getSession } from './utilities/getSession.js'
 import { handleError } from './utilities/handleError.js'
-import { sanitizeRelationshipIDs } from './utilities/sanitizeRelationshipIDs.js'
+import { transform } from './utilities/transform.js'
 
 export const create: Create = async function create(
   this: MongooseAdapter,
   { collection, data, req },
 ) {
   const Model = this.collections[collection]
-  const options: CreateOptions = {
-    session: await getSession(this, req),
-  }
-
-  let doc
+  const session = await getSession(this, req)
 
-  const sanitizedData = sanitizeRelationshipIDs({
-    config: this.payload.config,
-    data,
-    fields: this.payload.collections[collection].config.fields,
-  })
+  const fields = this.payload.collections[collection].config.flattenedFields
 
   if (this.payload.collections[collection].customIDType) {
-    sanitizedData._id = sanitizedData.id
+    data._id = data.id
   }
 
+  transform({
+    adapter: this,
+    data,
+    fields,
+    operation: 'create',
+  })
+
   try {
-    ;[doc] = await Model.create([sanitizedData], options)
-  } catch (error) {
-    handleError({ collection, error, req })
-  }
+    const { insertedId } = await Model.collection.insertOne(data, { session })
+    data._id = insertedId
 
-  // doc.toJSON does not do stuff like converting ObjectIds to string, or date strings to date objects. That's why we use JSON.parse/stringify here
-  const result: Document = JSON.parse(JSON.stringify(doc))
-  const verificationToken = doc._verificationToken
+    transform({
+      adapter: this,
+      data,
+      fields,
+      operation: 'read',
+    })
 
-  // custom id type reset
-  result.id = result._id
-  if (verificationToken) {
-    result._verificationToken = verificationToken
+    return data
+  } catch (error) {
+    handleError({ collection, error, req })
   }
-
-  return result
 }

packages/db-mongodb/src/createGlobal.ts

@@ -1,38 +1,39 @@
-import type { CreateOptions } from 'mongoose'
 import type { CreateGlobal } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
 import { getSession } from './utilities/getSession.js'
-import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
-import { sanitizeRelationshipIDs } from './utilities/sanitizeRelationshipIDs.js'
+import { transform } from './utilities/transform.js'
 
 export const createGlobal: CreateGlobal = async function createGlobal(
   this: MongooseAdapter,
   { slug, data, req },
 ) {
   const Model = this.globals
 
-  const global = sanitizeRelationshipIDs({
-    config: this.payload.config,
-    data: {
-      globalType: slug,
-      ...data,
-    },
-    fields: this.payload.config.globals.find((globalConfig) => globalConfig.slug === slug).fields,
-  })
+  const fields = this.payload.config.globals.find(
+    (globalConfig) => globalConfig.slug === slug,
+  ).flattenedFields
 
-  const options: CreateOptions = {
-    session: await getSession(this, req),
-  }
+  transform({
+    adapter: this,
+    data,
+    fields,
+    globalSlug: slug,
+    operation: 'create',
+  })
 
-  let [result] = (await Model.create([global], options)) as any
+  const session = await getSession(this, req)
 
-  result = JSON.parse(JSON.stringify(result))
+  const { insertedId } = await Model.collection.insertOne(data, { session })
+  ;(data as any)._id = insertedId
 
-  // custom id type reset
-  result.id = result._id
-  result = sanitizeInternalFields(result)
+  transform({
+    adapter: this,
+    data,
+    fields,
+    operation: 'read',
+  })
 
-  return result
+  return data
 }