perf(db-mongodb): faster join field aggregation by replacing mongoose-aggregate-paginate-v2 with a custom implementation (#10936)

Fixes
#10165 (reply in thread)

As described in the discussion, we have an incorrect order of
aggregation pipeline when using aggregations with the join field. We
must use `$sort`, `$skip`, `$limit` before the `$lookup` or otherwise
mongodb scans all the docs, applies `$lookup` for them and only after
applies `$limit`, `$skip`.
Replaces `mongoose-aggregate-paginate-v2` with a custom
`aggregatePaginate` because we need a custom solution here. This was
considered in #9594 but it was
reverted as for now.

Fixes #11187

Author: r1tsuu

packages/db-mongodb/package.json

@@ -47,14 +47,12 @@
   },
   "dependencies": {
     "mongoose": "8.9.5",
-    "mongoose-aggregate-paginate-v2": "1.1.2",
     "mongoose-paginate-v2": "1.8.5",
     "prompts": "2.4.2",
     "uuid": "10.0.0"
   },
   "devDependencies": {
     "@payloadcms/eslint-config": "workspace:*",
-    "@types/mongoose-aggregate-paginate-v2": "1.0.12",
     "mongodb": "6.12.0",
     "mongodb-memory-server": "^10",
     "payload": "workspace:*"

packages/db-mongodb/src/find.ts

@@ -7,6 +7,7 @@ import type { MongooseAdapter } from './index.js'
 
 import { buildQuery } from './queries/buildQuery.js'
 import { buildSortParam } from './queries/buildSortParam.js'
+import { aggregatePaginate } from './utilities/aggregatePaginate.js'
 import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
 import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { getSession } from './utilities/getSession.js'
@@ -128,7 +129,20 @@ export const find: Find = async function find(
   })
   // build join aggregation
   if (aggregate) {
-    result = await Model.aggregatePaginate(Model.aggregate(aggregate), paginationOptions)
+    result = await aggregatePaginate({
+      adapter: this,
+      collation: paginationOptions.collation,
+      joinAggregation: aggregate,
+      limit: paginationOptions.limit,
+      Model,
+      page: paginationOptions.page,
+      pagination: paginationOptions.pagination,
+      projection: paginationOptions.projection,
+      query,
+      session: paginationOptions.options?.session,
+      sort: paginationOptions.sort as object,
+      useEstimatedCount: paginationOptions.useEstimatedCount,
+    })
   } else {
     result = await Model.paginate(query, paginationOptions)
   }

packages/db-mongodb/src/findOne.ts

@@ -4,6 +4,7 @@ import type { FindOne } from 'payload'
 import type { MongooseAdapter } from './index.js'
 
 import { buildQuery } from './queries/buildQuery.js'
+import { aggregatePaginate } from './utilities/aggregatePaginate.js'
 import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
 import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { getSession } from './utilities/getSession.js'
@@ -40,15 +41,24 @@ export const findOne: FindOne = async function findOne(
     collection,
     collectionConfig,
     joins,
-    limit: 1,
     locale,
     projection,
     query,
   })
 
   let doc
   if (aggregate) {
-    ;[doc] = await Model.aggregate(aggregate, { session })
+    const { docs } = await aggregatePaginate({
+      adapter: this,
+      joinAggregation: aggregate,
+      limit: 1,
+      Model,
+      pagination: false,
+      projection,
+      query,
+      session,
+    })
+    doc = docs[0]
   } else {
     ;(options as Record<string, unknown>).projection = projection
     doc = await Model.findOne(query, {}, options)

packages/db-mongodb/src/index.ts

@@ -92,7 +92,10 @@ export interface Args {
 
   /** Extra configuration options */
   connectOptions?: {
-    /** Set false to disable $facet aggregation in non-supporting databases, Defaults to true */
+    /**
+     * Set false to disable $facet aggregation in non-supporting databases, Defaults to true
+     * @deprecated Payload doesn't use `$facet` anymore anywhere.
+     */
     useFacet?: boolean
   } & ConnectOptions
   /** Set to true to disable hinting to MongoDB to use 'id' as index. This is currently done when counting documents for pagination. Disabling this optimization might fix some problems with AWS DocumentDB. Defaults to false */

packages/db-mongodb/src/init.ts

@@ -2,7 +2,6 @@ import type { PaginateOptions } from 'mongoose'
 import type { Init, SanitizedCollectionConfig } from 'payload'
 
 import mongoose from 'mongoose'
-import mongooseAggregatePaginate from 'mongoose-aggregate-paginate-v2'
 import paginate from 'mongoose-paginate-v2'
 import { buildVersionCollectionFields, buildVersionGlobalFields } from 'payload'
 
@@ -48,25 +47,21 @@ export const init: Init = function init(this: MongooseAdapter) {
         }),
       )
 
-      if (Object.keys(collection.joins).length > 0) {
-        versionSchema.plugin(mongooseAggregatePaginate)
-      }
-
       const versionCollectionName =
         this.autoPluralization === true && !collection.dbName ? undefined : versionModelName
 
       this.versions[collection.slug] = mongoose.model(
         versionModelName,
         versionSchema,
         versionCollectionName,
-      ) as CollectionModel
+      ) as unknown as CollectionModel
     }
 
     const modelName = getDBName({ config: collection })
     const collectionName =
       this.autoPluralization === true && !collection.dbName ? undefined : modelName
 
-    this.collections[collection.slug] = mongoose.model(
+    this.collections[collection.slug] = mongoose.model<any>(
       modelName,
       schema,
       collectionName,
@@ -101,7 +96,7 @@ export const init: Init = function init(this: MongooseAdapter) {
         }),
       )
 
-      this.versions[global.slug] = mongoose.model(
+      this.versions[global.slug] = mongoose.model<any>(
         versionModelName,
         versionSchema,
         versionModelName,

packages/db-mongodb/src/models/buildCollectionSchema.ts

@@ -1,7 +1,6 @@
 import type { PaginateOptions, Schema } from 'mongoose'
 import type { Payload, SanitizedCollectionConfig } from 'payload'
 
-import mongooseAggregatePaginate from 'mongoose-aggregate-paginate-v2'
 import paginate from 'mongoose-paginate-v2'
 
 import { getBuildQueryPlugin } from '../queries/getBuildQueryPlugin.js'
@@ -44,12 +43,5 @@ export const buildCollectionSchema = (
     .plugin<any, PaginateOptions>(paginate, { useEstimatedCount: true })
     .plugin(getBuildQueryPlugin({ collectionSlug: collection.slug }))
 
-  if (
-    Object.keys(collection.joins).length > 0 ||
-    Object.keys(collection.polymorphicJoins).length > 0
-  ) {
-    schema.plugin(mongooseAggregatePaginate)
-  }
-
   return schema
 }

packages/db-mongodb/src/queryDrafts.ts

@@ -7,6 +7,7 @@ import type { MongooseAdapter } from './index.js'
 
 import { buildQuery } from './queries/buildQuery.js'
 import { buildSortParam } from './queries/buildSortParam.js'
+import { aggregatePaginate } from './utilities/aggregatePaginate.js'
 import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
 import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { getSession } from './utilities/getSession.js'
@@ -116,10 +117,20 @@ export const queryDrafts: QueryDrafts = async function queryDrafts(
 
   // build join aggregation
   if (aggregate) {
-    result = await VersionModel.aggregatePaginate(
-      VersionModel.aggregate(aggregate),
-      paginationOptions,
-    )
+    result = await aggregatePaginate({
+      adapter: this,
+      collation: paginationOptions.collation,
+      joinAggregation: aggregate,
+      limit: paginationOptions.limit,
+      Model: VersionModel,
+      page: paginationOptions.page,
+      pagination: paginationOptions.pagination,
+      projection: paginationOptions.projection,
+      query: versionQuery,
+      session: paginationOptions.options?.session,
+      sort: paginationOptions.sort as object,
+      useEstimatedCount: paginationOptions.useEstimatedCount,
+    })
   } else {
     result = await VersionModel.paginate(versionQuery, paginationOptions)
   }

packages/db-mongodb/src/types.ts

@@ -1,12 +1,5 @@
 import type { ClientSession } from 'mongodb'
-import type {
-  AggregatePaginateModel,
-  IndexDefinition,
-  IndexOptions,
-  Model,
-  PaginateModel,
-  SchemaOptions,
-} from 'mongoose'
+import type { IndexDefinition, IndexOptions, Model, PaginateModel, SchemaOptions } from 'mongoose'
 import type {
   ArrayField,
   BlocksField,
@@ -37,10 +30,7 @@ import type {
 
 import type { BuildQueryArgs } from './queries/getBuildQueryPlugin.js'
 
-export interface CollectionModel
-  extends Model<any>,
-    PaginateModel<any>,
-    AggregatePaginateModel<any> {
+export interface CollectionModel extends Model<any>, PaginateModel<any> {
   /** buildQuery is used to transform payload's where operator into what can be used by mongoose (e.g. id => _id) */
   buildQuery: (args: BuildQueryArgs) => Promise<Record<string, unknown>> // TODO: Delete this
 }

packages/db-mongodb/src/utilities/aggregatePaginate.ts

@@ -0,0 +1,104 @@
+import type { CollationOptions } from 'mongodb'
+import type { ClientSession, Model, PipelineStage } from 'mongoose'
+import type { PaginatedDocs } from 'payload'
+
+import type { MongooseAdapter } from '../index.js'
+
+export const aggregatePaginate = async ({
+  adapter,
+  collation,
+  joinAggregation,
+  limit,
+  Model,
+  page,
+  pagination,
+  projection,
+  query,
+  session,
+  sort,
+  useEstimatedCount,
+}: {
+  adapter: MongooseAdapter
+  collation?: CollationOptions
+  joinAggregation?: PipelineStage[]
+  limit?: number
+  Model: Model<any>
+  page?: number
+  pagination?: boolean
+  projection?: Record<string, boolean>
+  query: Record<string, unknown>
+  session?: ClientSession
+  sort?: object
+  useEstimatedCount?: boolean
+}): Promise<PaginatedDocs<any>> => {
+  const aggregation: PipelineStage[] = [{ $match: query }]
+
+  if (sort) {
+    const $sort: Record<string, -1 | 1> = {}
+
+    Object.entries(sort).forEach(([key, value]) => {
+      $sort[key] = value === 'desc' ? -1 : 1
+    })
+
+    aggregation.push({ $sort })
+  }
+
+  if (page) {
+    aggregation.push({ $skip: (page - 1) * (limit ?? 0) })
+  }
+
+  if (limit) {
+    aggregation.push({ $limit: limit })
+  }
+
+  if (joinAggregation) {
+    for (const stage of joinAggregation) {
+      aggregation.push(stage)
+    }
+  }
+
+  if (projection) {
+    aggregation.push({ $project: projection })
+  }
+
+  let countPromise: Promise<null | number> = Promise.resolve(null)
+
+  if (pagination !== false && limit) {
+    if (useEstimatedCount) {
+      countPromise = Model.estimatedDocumentCount(query)
+    } else {
+      const hint = adapter.disableIndexHints !== true ? { _id: 1 } : undefined
+      countPromise = Model.countDocuments(query, { collation, hint, session })
+    }
+  }
+
+  const [docs, countResult] = await Promise.all([
+    Model.aggregate(aggregation, { collation, session }),
+    countPromise,
+  ])
+
+  const count = countResult === null ? docs.length : countResult
+
+  const totalPages =
+    pagination !== false && typeof limit === 'number' && limit !== 0 ? Math.ceil(count / limit) : 1
+
+  const hasPrevPage = pagination !== false && page > 1
+  const hasNextPage = pagination !== false && totalPages > page
+  const pagingCounter =
+    pagination !== false && typeof limit === 'number' ? (page - 1) * limit + 1 : 1
+
+  const result: PaginatedDocs = {
+    docs,
+    hasNextPage,
+    hasPrevPage,
+    limit,
+    nextPage: hasNextPage ? page + 1 : null,
+    page,
+    pagingCounter,
+    prevPage: hasPrevPage ? page - 1 : null,
+    totalDocs: count,
+    totalPages,
+  }
+
+  return result
+}

packages/db-mongodb/src/utilities/buildJoinAggregation.ts

@@ -19,8 +19,6 @@ type BuildJoinAggregationArgs = {
   collection: CollectionSlug
   collectionConfig: SanitizedCollectionConfig
   joins: JoinQuery
-  // the number of docs to get at the top collection level
-  limit?: number
   locale: string
   projection?: Record<string, true>
   // the where clause for the top collection
@@ -34,10 +32,8 @@ export const buildJoinAggregation = async ({
   collection,
   collectionConfig,
   joins,
-  limit,
   locale,
   projection,
-  query,
   versions,
 }: BuildJoinAggregationArgs): Promise<PipelineStage[] | undefined> => {
   if (
@@ -49,24 +45,8 @@ export const buildJoinAggregation = async ({
   }
 
   const joinConfig = adapter.payload.collections[collection].config.joins
+  const aggregate: PipelineStage[] = []
   const polymorphicJoinsConfig = adapter.payload.collections[collection].config.polymorphicJoins
-  const aggregate: PipelineStage[] = [
-    {
-      $sort: { createdAt: -1 },
-    },
-  ]
-
-  if (query) {
-    aggregate.push({
-      $match: query,
-    })
-  }
-
-  if (limit) {
-    aggregate.push({
-      $limit: limit,
-    })
-  }
 
   for (const join of polymorphicJoinsConfig) {
     if (projection && !projection[join.joinPath]) {
@@ -448,9 +428,5 @@ export const buildJoinAggregation = async ({
     }
   }
 
-  if (projection) {
-    aggregate.push({ $project: projection })
-  }
-
   return aggregate
 }