From 6faffe86be3a2b9f8538e48eff403d8218f8bb92 Mon Sep 17 00:00:00 2001 From: lau1944 Date: Sun, 26 Jun 2022 23:32:52 +0800 Subject: [PATCH 1/4] feat: add sub document pagination (feats: #174) --- index.d.ts | 43 +++++++++-- src/index.js | 203 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/index.js | 71 ++++++++++++++++- 3 files changed, 308 insertions(+), 9 deletions(-) diff --git a/index.d.ts b/index.d.ts index 0f6f4a2..3c70120 100644 --- a/index.d.ts +++ b/index.d.ts @@ -23,12 +23,12 @@ declare module 'mongoose' { collation?: import('mongodb').CollationOptions | undefined; sort?: object | string | undefined; populate?: - | PopulateOptions[] - | string[] - | PopulateOptions - | string - | PopulateOptions - | undefined; + | PopulateOptions[] + | string[] + | PopulateOptions + | string + | PopulateOptions + | undefined; projection?: any; lean?: boolean | undefined; leanWithId?: boolean | undefined; @@ -46,6 +46,32 @@ declare module 'mongoose' { options?: QueryOptions | undefined; } + interface SubPaginateOptions { + select?: object | string | undefined; + populate?: + | PopulateOptions[] + | string[] + | PopulateOptions + | string + | PopulateOptions + | undefined; + pagination?: boolean | undefined; + read?: ReadOptions | undefined; + pagingOptions: SubDocumentPagingOptions | undefined; + } + + interface SubDocumentPagingOptions { + populate?: + | PopulateOptions[] + | string[] + | PopulateOptions + | string + | PopulateOptions + | undefined; + page?: number | undefined; + limit?: number | undefined; + } + interface PaginateResult { docs: T[]; totalDocs: number; @@ -69,8 +95,8 @@ declare module 'mongoose' { O extends PaginateOptions = {} > = O['lean'] extends true ? O['leanWithId'] extends true - ? LeanDocument - : LeanDocument + ? LeanDocument + : LeanDocument : HydratedDocument; interface PaginateModel @@ -91,6 +117,7 @@ declare function _(schema: mongoose.Schema): void; export = _; declare namespace _ { const paginate: { options: mongoose.PaginateOptions }; + const paginateSubDocs: { options: mongoose.PaginateOptions }; class PaginationParameters { constructor(request: { query?: Record }); getOptions: () => O; diff --git a/src/index.js b/src/index.js index f1c37bf..7784fb8 100644 --- a/src/index.js +++ b/src/index.js @@ -283,12 +283,215 @@ function paginate(query, options, callback) { }); } +/** + * Pagination process for sub-documents + * internally, it would call `query.findOne`, return only one document + * + * @param {Object} query + * @param {Object} options + * @param {Function} callback + */ +function paginateSubDocs(query, options, callback) { + /** + * Populate sub documents with pagination fields + * + * @param {Object} query + * @param {Object} populate origin populate option + * @param {Object} option + */ + function getSubDocsPopulate(option) { + /** + * options properties for sub-documents pagination + * + * @param {String} populate: populate option for sub documents + * @param {Number} page + * @param {Number} limit + * + * @returns {String} countLabel + */ + let { populate, page = 1, limit = 10 } = option; + + if (!populate) { + throw new Error('populate is required'); + } + + const offset = (page - 1) * limit; + option.offset = offset; + const pagination = { + skip: offset, + limit: limit, + }; + + if (typeof populate === 'string') { + populate = { + path: populate, + ...pagination, + }; + } else if (typeof populate === 'object' && !Array.isArray(populate)) { + populate = Object.assign(populate, pagination); + } + option.populate = populate; + + return populate; + } + + function populateResult(result, populate, callback) { + return result.populate(populate, callback); + } + + /** + * Convert result of sub-docs list to pagination like docs + * + * @param {Object} result query result + * @param {Object} option pagination option + */ + function constructDocs(paginatedResult, option) { + let { populate, offset = 0, page = 1, limit = 10 } = option; + + const path = populate.path; + const count = option.count; + const paginatedDocs = paginatedResult[path]; + + if (!paginatedDocs) { + throw new Error( + `Parse error! Cannot find key on result with path ${path}` + ); + } + + page = Math.ceil((offset + 1) / limit); + + // set default meta + const meta = { + docs: paginatedDocs, + totalDocs: count || 1, + limit: limit, + page: page, + prevPage: null, + nextPage: null, + hasPrevPage: false, + hasNextPage: false, + }; + + const totalPages = limit > 0 ? Math.ceil(count / limit) || 1 : null; + meta.totalPages = totalPages; + meta.pagingCounter = (page - 1) * limit + 1; + + // Set prev page + if (page > 1) { + meta.hasPrevPage = true; + meta.prevPage = page - 1; + } else if (page == 1 && offset !== 0) { + meta.hasPrevPage = true; + meta.prevPage = 1; + } + + // Set next page + if (page < totalPages) { + meta.hasNextPage = true; + meta.nextPage = page + 1; + } + + if (limit == 0) { + meta.limit = 0; + meta.totalPages = 1; + meta.page = 1; + meta.pagingCounter = 1; + } + + Object.defineProperty(paginatedResult, path, { + value: meta, + writable: false, + }); + } + + options = Object.assign(options, { + customLabels: defaultOptions.customLabels, + }); + + // options properties for main document query + const { + populate, + read = {}, + select = '', + pagination = true, + pagingOptions, + } = options; + + const mQuery = this.findOne(query, options.projection); + + if (read && read.pref) { + /** + * Determines the MongoDB nodes from which to read. + * @param read.pref one of the listed preference options or aliases + * @param read.tags optional tags for this query + */ + mQuery.read(read.pref, read.tags); + } + + if (select) { + mQuery.select(select); + } + + return new Promise((resolve, reject) => { + mQuery + .exec() + .then((result) => { + let newPopulate = []; + + if (populate) { + newPopulate.push(newPopulate); + } + + if (pagination && pagingOptions) { + if (Array.isArray(pagingOptions)) { + pagingOptions.forEach((option) => { + let populate = getSubDocsPopulate(option); + option.count = result[populate.path].length; + newPopulate.push(populate); + }); + } else { + let populate = getSubDocsPopulate(pagingOptions); + pagingOptions.count = result[populate.path].length; + newPopulate.push(populate); + } + } + + populateResult(result, newPopulate, (err, paginatedResult) => { + if (err) { + callback?.(err, null); + reject(err); + return; + } + // convert paginatedResult to pagination docs + if (pagination && pagingOptions) { + if (Array.isArray(pagingOptions)) { + pagingOptions.forEach((option) => { + constructDocs(paginatedResult, option); + }); + } else { + constructDocs(paginatedResult, pagingOptions); + } + } + + callback?.(null, paginatedResult); + resolve(paginatedResult); + }); + }) + .catch((err) => { + console.error(err.message); + callback?.(err, null); + }); + }); +} + /** * @param {Schema} schema */ module.exports = (schema) => { schema.statics.paginate = paginate; + schema.statics.paginateSubDocs = paginateSubDocs; }; module.exports.PaginationParameters = PaginationParametersHelper; module.exports.paginate = paginate; +module.exports.paginateSubDocs = paginateSubDocs; diff --git a/tests/index.js b/tests/index.js index ee64f47..f0a6904 100644 --- a/tests/index.js +++ b/tests/index.js @@ -8,10 +8,18 @@ let PaginationParameters = require('../dist/pagination-parameters'); let MONGO_URI = 'mongodb://localhost/mongoose_paginate_test'; +let UserSchema = new mongoose.Schema({ + name: String, + age: Number, + gender: Number, +}); + let AuthorSchema = new mongoose.Schema({ name: String, }); + let Author = mongoose.model('Author', AuthorSchema); +let User = mongoose.model('User', UserSchema); let BookSchema = new mongoose.Schema({ title: String, @@ -21,6 +29,12 @@ let BookSchema = new mongoose.Schema({ type: mongoose.Schema.ObjectId, ref: 'Author', }, + used: [ + { + type: mongoose.Schema.ObjectId, + ref: 'User', + }, + ], loc: Object, }); @@ -87,11 +101,23 @@ describe('mongoose-paginate', function () { mongoose.connection.db.dropDatabase(done); }); - before(function () { + before(async function () { let book, books = []; let date = new Date(); + // create users + let users = []; + for (let i = 0; i < 10; ++i) { + const user = new User({ + name: randomString(), + gender: 1, + age: i, + }); + const newUser = await User.create(user); + users.push(newUser); + } + return Author.create({ name: 'Arthur Conan Doyle', }).then(function (author) { @@ -102,6 +128,7 @@ describe('mongoose-paginate', function () { title: 'Book #' + i, date: new Date(date.getTime() + i), author: author._id, + used: users, loc: { type: 'Point', coordinates: [-10.97, 20.77], @@ -505,6 +532,33 @@ describe('mongoose-paginate', function () { expect(result.meta.total).to.equal(100); }); }); + + it('Sub documents pagination', () => { + var query = { title: 'Book #1' }; + var option = { + pagingOptions: { + populate: { + path: 'used', + }, + page: 2, + limit: 3, + }, + }; + + return Book.paginateSubDocs(query, option).then((result) => { + expect(result.used.docs).to.have.length(3); + expect(result.used.totalPages).to.equal(4); + expect(result.used.page).to.equal(2); + expect(result.used.limit).to.equal(3); + expect(result.used.hasPrevPage).to.equal(true); + expect(result.used.hasNextPage).to.equal(true); + expect(result.used.prevPage).to.equal(1); + expect(result.used.nextPage).to.equal(3); + expect(result.used.pagingCounter).to.equal(4); + expect(result.used.docs[0].age).to.equal(3); + }); + }); + /* it('2dsphere', function () { var query = { @@ -710,3 +764,18 @@ describe('mongoose-paginate', function () { mongoose.disconnect(done); }); }); + +function randomString(strLength, charSet) { + var result = []; + + strLength = strLength || 5; + charSet = + charSet || 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + + while (strLength--) { + // (note, fixed typo) + result.push(charSet.charAt(Math.floor(Math.random() * charSet.length))); + } + + return result.join(''); +} From 4a3cc987cb563312e5e0340d0e6726ce2e913458 Mon Sep 17 00:00:00 2001 From: lau1944 Date: Sun, 26 Jun 2022 23:42:36 +0800 Subject: [PATCH 2/4] docs: update documentation for sub-doc pagination method (feats: #174) --- README.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 677f032..427ed85 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Prior to version `1.5.0`, types need to be installed from [DefinitelyTyped](http To declare a `PaginateModel` in your Typescript files: ```ts -import mongoose from 'mongoose' +import mongoose from 'mongoose'; import paginate from 'mongoose-paginate-v2'; // declare your schema @@ -379,6 +379,28 @@ Model.paginate({}, options, function (err, result) { }); ``` +### Pagination for sub documents + +If you want to paginate your sub-documents, here is the method you can use. + +```js +var query = { name: 'John' } +var option = { + select: 'name follower' + pagingOptions: { + // your populate option + populate: { + path: 'follower', + }, + page: 2, + limit: 10, + }, +}; + +// Only one document (which object key with name John) will be return +const result = await Book.paginateSubDocs(query, option); +``` + Below are some references to understand more about preferences, - https://github.com/Automattic/mongoose/blob/master/lib/query.js#L1008 From b02a57970bda2521e0e5b5b2a07b0b93d71f8fdb Mon Sep 17 00:00:00 2001 From: 1au <40358357+lau1944@users.noreply.github.com> Date: Sun, 26 Jun 2022 23:46:16 +0800 Subject: [PATCH 3/4] docs: update doc (feats: #174) --- README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 427ed85..8d5e16e 100644 --- a/README.md +++ b/README.md @@ -361,24 +361,6 @@ Model.paginate({}, options, function (err, result) { }); ``` -#### AllowDiskUse for large datasets - -Sets the allowDiskUse option, which allows the MongoDB server to use more than 100 MB for query. This option can let you work around `QueryExceededMemoryLimitNoDiskUseAllowed` errors from the MongoDB server. - -**Note that this option requires MongoDB server >= 4.4. Setting this option is a no-op for MongoDB 4.2 and earlier.** - -```js -const options = { - limit: 10, - page: 1, - allowDiskUse: true, -}; - -Model.paginate({}, options, function (err, result) { - // Result -}); -``` - ### Pagination for sub documents If you want to paginate your sub-documents, here is the method you can use. @@ -401,6 +383,24 @@ var option = { const result = await Book.paginateSubDocs(query, option); ``` +#### AllowDiskUse for large datasets + +Sets the allowDiskUse option, which allows the MongoDB server to use more than 100 MB for query. This option can let you work around `QueryExceededMemoryLimitNoDiskUseAllowed` errors from the MongoDB server. + +**Note that this option requires MongoDB server >= 4.4. Setting this option is a no-op for MongoDB 4.2 and earlier.** + +```js +const options = { + limit: 10, + page: 1, + allowDiskUse: true, +}; + +Model.paginate({}, options, function (err, result) { + // Result +}); +``` + Below are some references to understand more about preferences, - https://github.com/Automattic/mongoose/blob/master/lib/query.js#L1008 From 3284fe32ef9b853130bb583a28381f23cdadf24f Mon Sep 17 00:00:00 2001 From: lau1944 Date: Sun, 26 Jun 2022 23:56:42 +0800 Subject: [PATCH 4/4] feat: remove unrelated code comment --- src/index.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/index.js b/src/index.js index 7784fb8..bc78e43 100644 --- a/src/index.js +++ b/src/index.js @@ -300,15 +300,7 @@ function paginateSubDocs(query, options, callback) { * @param {Object} option */ function getSubDocsPopulate(option) { - /** - * options properties for sub-documents pagination - * - * @param {String} populate: populate option for sub documents - * @param {Number} page - * @param {Number} limit - * - * @returns {String} countLabel - */ + // options properties for sub-documents pagination let { populate, page = 1, limit = 10 } = option; if (!populate) {