From c913ed8f907cc151b3445e416d92bfc8fe97750a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Demonte Date: Sat, 27 Feb 2016 23:40:14 +0100 Subject: [PATCH] add hydratation feature --- README.md | 39 +++++++++ index.js | 58 ++++++++++++-- package.json | 2 +- test/hydratation.js | 189 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 test/hydratation.js diff --git a/README.md b/README.md index 0aa8070..3e488af 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ mongoose-elasticsearch-xp is a [mongoose](http://mongoosejs.com/) plugin that ca - [Indexing on demand](#indexing-on-demand) - [Mapping](#mapping) - [Creating mappings on-demand](#creating-mappings-on-demand) +- [Hydration](#hydration) ## Why this plugin? @@ -44,6 +45,7 @@ Options are: * `port` - the port Elasticsearch is running on * `auth` - the authentication needed to reach Elasticsearch server. In the standard format of 'username:password' * `protocol` - the protocol the Elasticsearch server uses. Defaults to http +* `hydrate` - whether or not to replace ES source by mongo document To have a model indexed into Elasticsearch simply add the plugin. @@ -295,6 +297,43 @@ User You'll have to manage whether or not you need to create the mapping, mongoose-elasticsearch-xp will make no assumptions and simply attempt to create the mapping. If the mapping already exists, an Exception detailing such will be populated in the `err` argument. +## Hydration +By default objects returned from performing a search will be the objects as is in Elasticsearch. +This is useful in cases where only what was indexed needs to be displayed (think a list of results) while the actual mongoose object contains the full data when viewing one of the results. + +However, if you want the results to be actual mongoose objects you can provide {hydrate:true} as the second argument to a search call. + +```javascript +User + .esSearch({query_string: {query: "john"}}, {hydrate:true}) + .then(function (results) { + // results here + }); +``` + +To modify default hydratation, provide an object to `hydrate` instead of "true". +`hydrate` accept {select: string, options: object, docsOnly: boolean} + +```javascript +User + .esSearch({query_string: {query: "john"}}, {hydrate: {select: 'name age', options: {lean: true}}}) + .then(function (results) { + // results here + }); +``` + +When using hydration, `hits._source` is replaced by `hits.doc`. + +If you only want the models, instead of the complete ES results, use the option "docsOnly". + +```javascript +User + .esSearch({query_string: {query: "john"}}, {hydrate: {select: 'name age', docsOnly; true}}) + .then(function (users) { + // users is an array of User + }); +``` + [npm-url]: https://npmjs.org/package/mongoose-elasticsearch-xp [npm-image]: https://badge.fury.io/js/mongoose-elasticsearch-xp.svg diff --git a/index.js b/index.js index 050497c..9dd1b2b 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ var generateMapping = require('./lib/mapping').generate; var client = require('./lib/client'); var utils = require('./lib/utils'); var Bulker = require('./lib/bulker'); +var mongoose = require('mongoose'); module.exports = function (schema, options) { @@ -146,7 +147,8 @@ function search(query, options, callback) { query = query || {}; options = options || {}; - var esOptions = this.esOptions(); + var self = this; + var esOptions = self.esOptions(); var params = { index: esOptions.index, type: esOptions.type @@ -158,7 +160,51 @@ function search(query, options, callback) { } else { params.body = query.query ? query : {query: query}; } - esOptions.client.search(params, defer.callback); + if (options.hydrate) { + params._source = false; + } + esOptions.client.search(params, function (err, result) { + if (err) { + return defer.reject(err); + } + if (!options.hydrate) { + return defer.resolve(result); + } + if (!result.hits.total) { + return defer.resolve(result); + } + + var ids = result.hits.hits.map(function (hit) { + return mongoose.Types.ObjectId(hit._id); + }); + + var hydrate = options.hydrate || {}; + var select = hydrate.select || null; + var opts = hydrate.options || null; + var docsOnly = hydrate.docsOnly || false; + + + self.find({_id: {$in: ids}}, select, opts, function (err, users) { + if (err) { + return defer.reject(err); + } + var userByIds = {}; + users.forEach(function (user) { + userByIds[user._id] = user; + }); + if (docsOnly) { + result = ids.map(function (id) { + return userByIds[id]; + }); + } else { + result.hits.hits.forEach(function (hit) { + hit.doc = userByIds[hit._id]; + }); + } + return defer.resolve(result); + }); + + }); return defer.promise; } @@ -187,7 +233,7 @@ function synchronize(conditions, projection, options, callback) { options = null; } - var schema = this; + var model = this; var defer = utils.defer(callback); var esOptions = this.esOptions(); var batch = esOptions.bulk && esOptions.bulk.batch ? esOptions.bulk.batch : 50; @@ -202,7 +248,7 @@ function synchronize(conditions, projection, options, callback) { } function onError(err) { - schema.emit('es-bulk-error', err); + model.emit('es-bulk-error', err); if (streamClosed) { finalize(); } else { @@ -211,7 +257,7 @@ function synchronize(conditions, projection, options, callback) { } function onSent(len) { - schema.emit('es-bulk-sent', len); + model.emit('es-bulk-sent', len); if (streamClosed) { finalize(); } else { @@ -228,7 +274,7 @@ function synchronize(conditions, projection, options, callback) { {index: {_index: esOptions.index, _type: esOptions.type, _id: doc._id.toString()}}, utils.serialize(doc, esOptions.mapping) ); - schema.emit('es-bulk-data', doc); + model.emit('es-bulk-data', doc); if (!sending) { stream.resume(); } diff --git a/package.json b/package.json index 50fdd3d..9dfefea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mongoose-elasticsearch-xp", - "version": "1.2.0", + "version": "1.3.0", "description": "A mongoose plugin that indexes models into elastic search (an alternative to mongoosastic)", "tags": [ "mongodb", diff --git a/test/hydratation.js b/test/hydratation.js new file mode 100644 index 0000000..352fbe5 --- /dev/null +++ b/test/hydratation.js @@ -0,0 +1,189 @@ +var utils = require('./utils'); +var mongoose = require('mongoose'); +var plugin = require('../'); + +describe("hydratation", function () { + + utils.setup(); + + beforeEach(function (done) { + + var UserSchema = new mongoose.Schema({ + name: String, + age: Number + }); + + UserSchema.plugin(plugin); + + var UserModel = mongoose.model('User', UserSchema); + + var john = new UserModel({name: 'John', age: 35}); + var jane = new UserModel({name: 'Jane', age: 34}); + var bob = new UserModel({name: 'Bob', age: 36}); + + this.model = UserModel; + this.users = { + john: john, + jane: jane, + bob: bob + }; + + utils.deleteModelIndexes(UserModel) + .then(function () { + return UserModel.esCreateMapping(); + }) + .then(function () { + return utils.Promise.all([john, jane, bob].map(function (user) { + return new utils.Promise(function (resolve) { + user.on('es-indexed', resolve); + user.save(); + }); + })); + }) + .then(function () { + return UserModel.esRefresh(); + }) + .then(function () { + done(); + }); + }); + + it('should hydrate results', function (done) { + + var UserModel = this.model; + var john = this.users.john; + var bob = this.users.bob; + + UserModel + .esSearch( + { + query: {match_all: {}}, + sort: [ + {age: {order: "desc"}} + ], + filter: {range: {age: {gte: 35}}} + }, + {hydrate: true} + ) + .then(function (result) { + var hit; + expect(result.hits.total).to.eql(2); + + hit = result.hits.hits[0]; + expect(hit._source).to.be.undefined; + expect(hit.doc).to.be.an.instanceof(UserModel); + expect(hit.doc._id.toString()).to.eql(bob._id.toString()); + expect(hit.doc.name).to.eql(bob.name); + expect(hit.doc.age).to.eql(bob.age); + + hit = result.hits.hits[1]; + expect(hit._source).to.be.undefined; + expect(hit.doc).to.be.an.instanceof(UserModel); + expect(hit.doc._id.toString()).to.eql(john._id.toString()); + expect(hit.doc.name).to.eql(john.name); + expect(hit.doc.age).to.eql(john.age); + + done(); + }) + .catch(function (err) { + done(err); + }); + }); + + it('should hydrate returning only models', function (done) { + + var UserModel = this.model; + var john = this.users.john; + var bob = this.users.bob; + + UserModel + .esSearch( + { + query: {match_all: {}}, + sort: [ + {age: {order: "desc"}} + ], + filter: {range: {age: {gte: 35}}} + }, + {hydrate: {docsOnly: true}} + ) + .then(function (users) { + var user; + expect(users.length).to.eql(2); + + user = users[0]; + expect(user._id.toString()).to.eql(bob._id.toString()); + expect(user.name).to.eql(bob.name); + expect(user.age).to.eql(bob.age); + + user = users[1]; + expect(user._id.toString()).to.eql(john._id.toString()); + expect(user.name).to.eql(john.name); + expect(user.age).to.eql(john.age); + + done(); + }) + .catch(function (err) { + done(err); + }); + }); + + it('should hydrate using projection', function (done) { + + var UserModel = this.model; + var jane = this.users.jane; + + return UserModel + + .esSearch( + 'name:jane', + {hydrate: {select: 'name'}} + ) + .then(function (result) { + var hit; + expect(result.hits.total).to.eql(1); + + hit = result.hits.hits[0]; + expect(hit._source).to.be.undefined; + expect(hit.doc).to.be.an.instanceof(UserModel); + expect(hit.doc._id.toString()).to.eql(jane._id.toString()); + expect(hit.doc.name).to.eql(jane.name); + expect(hit.doc.age).to.be.undefined; + + done(); + }) + .catch(function (err) { + done(err); + }); + }); + + it('should hydrate using options', function (done) { + + var UserModel = this.model; + var jane = this.users.jane; + + return UserModel + + .esSearch( + 'name:jane', + {hydrate: {options: {lean: true}}} + ) + .then(function (result) { + var hit; + expect(result.hits.total).to.eql(1); + + hit = result.hits.hits[0]; + expect(hit._source).to.be.undefined; + expect(hit.doc).not.to.be.an.instanceof(UserModel); + expect(hit.doc._id.toString()).to.eql(jane._id.toString()); + expect(hit.doc.name).to.eql(jane.name); + expect(hit.doc.age).to.eql(jane.age); + + done(); + }) + .catch(function (err) { + done(err); + }); + }); + +});