Merge pull request #538 from dadi/feature/improved-search

Improve search

.travis.yml

@@ -4,6 +4,8 @@ cache:
     - node_modules
 notifications:
   email: false
+  slack:
+    secure: jnmTBfHGzU4rR9RAVsRmwMI3Rdul5PEKioh2oIsWv245DQrqr5DIwEONPRMsCbs1Duk69jiQFAzfLgzjtOjHleac79p0076pSxaG2zXNyzgya5YBMgYv6wrFh9eNZJoXyMvDw8jcipbRbx/Xp05aayClY39BJ7UUFlklvIeFb5wFqRfIlVXbLu5MFFatwCzB1IOeSeD90ZTX9VsWw97SJoJdta5gn7dx0P06SStPSJNOrLmV1/aXrSorBYmeTNJgUWPg4zHanocll8b7TR1a8Iyc5JBip0tEUpf1vJOMb/7bMdmNJ1XL8rzWw/87JujuzdOKikrsi52fzHE70TgsEBf0+BkwtFK2SzRvrwIKVesjj+ek/E2P1gal2qeqDVLVGZYuKAqbigv/0r6MHDcqJLoWb9MGHTLgcobZOfX3+UZetIc8dDA9/yme1auC8efrD4LOdo4Glhf6EZP4xCFUJpZive3VCKQfg/sMBaWAMWY508cxV+ey+fucV8fEqIsZHNPv7mCWoeMc5PnMd5TlR49UDgNrq0xcS8q2n9EKdZZARdsnqwdpmwJ3Ygd3gPsQTdYKsIBUxEoQKhMycO1XaqkbcUSHi8nSw85775cdEUscEob01WRN7D9IX7sGGp3LksFeR1CtjigBtMM9gHwmODUcPmcD+YRPYnnquoGzV9I=
 node_js:
   - '8'
   - '10'

config.js

@@ -135,7 +135,7 @@ var conf = convict({
     hashSecrets: {
       doc: 'Whether to hash client secrets',
       format: Boolean,
-      default: false
+      default: true
     },
     saltRounds: {
       doc: 'The number of rounds to go through when hashing a password',
@@ -144,11 +144,27 @@ var conf = convict({
     }
   },
   search: {
+    database: {
+      doc: 'The name of the database to use for storing and querying indexed documents',
+      format: String,
+      default: 'search',
+      env: 'DB_SEARCH_NAME'
+    },
+    datastore: {
+      doc: 'The datastore to use for storing and querying indexed documents',
+      format: String,
+      default: '@dadi/api-mongodb'
+    },
     enabled: {
       doc: 'If true, API responds to collection /search endpoints',
       format: Boolean,
       default: false
     },
+    indexCollection: {
+      doc: 'The name of the datastore collection that will hold the index of word matches',
+      format: String,
+      default: 'searchIndex'
+    },    
     minQueryLength: {
       doc: 'Minimum search string length',
       format: Number,
@@ -157,18 +173,7 @@ var conf = convict({
     wordCollection: {
       doc: 'The name of the datastore collection that will hold tokenized words',
       format: String,
-      default: 'words'
-    },
-    datastore: {
-      doc: 'The datastore to use for storing and querying indexed documents',
-      format: String,
-      default: '@dadi/api-mongodb'
-    },
-    database: {
-      doc: 'The name of the database to use for storing and querying indexed documents',
-      format: String,
-      default: 'search',
-      env: 'DB_SEARCH_NAME'
+      default: 'searchWords'
     }
   },
   caching: {
@@ -432,6 +437,18 @@ var conf = convict({
       format: Boolean,
       default: true
     }
+  },
+  workQueue: {
+    debounceTime: {
+      doc: 'The amount of idle time (in ms) required for the work queue to start a background job',
+      format: Number,
+      default: 500
+    },
+    pollingTime: {
+      doc: 'The interval (in ms) at which the work queue checks for new background jobs',
+      format: Number,
+      default: 200
+    }
   }
 })
 

config/config.test.json.sample

@@ -56,5 +56,9 @@
   },
   "feedback": false,
   "cors": false,
-  "cluster": false
+  "cluster": false,
+  "workQueue": {
+    "debounceTime": 0,
+    "pollingTime": 0
+  }
 }

dadi/lib/controller/collections.js

@@ -30,7 +30,7 @@ Collections.prototype.get = function (req, res, next) {
         return false
       }
 
-      let aclKey = this.server.components[key].model.aclKey
+      let aclKey = this.server.components[key].model.getAclKey()
 
       if (!clientIsAdmin && (!access[aclKey] || !access[aclKey].read)) {
         return false

dadi/lib/controller/documents.js

@@ -3,7 +3,10 @@ const config = require('./../../../config')
 const Controller = require('./index')
 const debug = require('debug')('api:controller')
 const help = require('./../help')
+const model = require('../model')
+const searchModel = require('./../model/search')
 const url = require('url')
+const workQueue = require('../workQueue')
 
 const Collection = function (model, server) {
   if (!model) throw new Error('Model instance required')
@@ -14,7 +17,7 @@ const Collection = function (model, server) {
 
 Collection.prototype = new Controller()
 
-Collection.prototype.count = function (req, res, next) {
+Collection.prototype.count = workQueue.wrapForegroundJob(function (req, res, next) {
   let method = req.method && req.method.toLowerCase()
 
   if (method !== 'get') {
@@ -41,9 +44,9 @@ Collection.prototype.count = function (req, res, next) {
   }).catch(error => {
     return help.sendBackJSON(null, res, next)(error)
   })
-}
+})
 
-Collection.prototype.delete = function (req, res, next) {
+Collection.prototype.delete = workQueue.wrapForegroundJob(function (req, res, next) {
   let query = req.params.id ? { _id: req.params.id } : req.body.query
 
   if (!query) return next()
@@ -79,9 +82,9 @@ Collection.prototype.delete = function (req, res, next) {
   }).catch(error => {
     return help.sendBackJSON(200, res, next)(error)
   })
-}
+})
 
-Collection.prototype.get = function (req, res, next) {
+Collection.prototype.get = workQueue.wrapForegroundJob(function (req, res, next) {
   let options = this._getURLParameters(req.url)
   let callback = options.callback || this.model.settings.callback
 
@@ -112,9 +115,9 @@ Collection.prototype.get = function (req, res, next) {
   }).catch(error => {
     return done(error)
   })
-}
+})
 
-Collection.prototype.post = function (req, res, next) {
+Collection.prototype.post = workQueue.wrapForegroundJob(function (req, res, next) {
   // Add internal fields.
   let internals = {
     _apiVersion: req.url.split('/')[1]
@@ -184,7 +187,7 @@ Collection.prototype.post = function (req, res, next) {
   }).catch(error => {
     return help.sendBackJSON(200, res, next)(error)
   })
-}
+})
 
 Collection.prototype.put = function (req, res, next) {
   return this.post(req, res, next)
@@ -239,7 +242,62 @@ Collection.prototype.registerRoutes = function (route, filePath) {
   })
 }
 
-Collection.prototype.stats = function (req, res, next) {
+Collection.prototype.search = function (req, res, next) {
+  const minimumQueryLength = config.get('search.minQueryLength')
+  const path = url.parse(req.url, true)
+  const {lang: language, q: query} = path.query
+  const {errors, queryOptions} = this._prepareQueryOptions(path.query)
+
+  if (!config.get('search.enabled')) {
+    const error = new Error('Not Implemented')
+
+    error.statusCode = 501
+    error.json = {
+      errors: [{
+        message: `Search is disabled or an invalid data connector has been specified.`
+      }]
+    }
+
+    return help.sendBackJSON(null, res, next)(error)
+  }
+
+  if (errors.length !== 0) {
+    return help.sendBackJSON(400, res, next)(null, queryOptions)
+  }
+
+  if (typeof query !== 'string' || query.length < minimumQueryLength) {
+    const error = new Error('Bad Request')
+
+    error.statusCode = 400
+    error.json = {
+      errors: [{
+        message: `Search query must be at least ${minimumQueryLength} characters.`
+      }]
+    }
+
+    return help.sendBackJSON(null, res, next)(error)
+  }
+
+  return this.model.validateAccess({
+    client: req.dadiApiClient,
+    type: 'read'
+  }).then(() => {
+    return searchModel.find({
+      client: req.dadiApiClient,
+      collections: [this.model.name],
+      fields: queryOptions.fields,
+      language,
+      modelFactory: model,
+      query
+    })
+  }).then(response => {
+    return help.sendBackJSON(200, res, next)(null, response)
+  }).catch(error => {
+    return help.sendBackJSON(null, res, next)(error)
+  })
+}
+
+Collection.prototype.stats = workQueue.wrapForegroundJob(function (req, res, next) {
   let method = req.method && req.method.toLowerCase()
 
   if (method !== 'get') {
@@ -253,14 +311,14 @@ Collection.prototype.stats = function (req, res, next) {
   }).catch(error => {
     return help.sendBackJSON(null, res, next)(error)
   })
-}
+})
 
 Collection.prototype.unregisterRoutes = function (route) {
   this.server.app.unuse(`${route}/config`)
   this.server.app.unuse(`${route}/:id(${this.ID_PATTERN})?/:action(count|search|stats|versions)?`)
 }
 
-Collection.prototype.versions = function (req, res, next) {
+Collection.prototype.versions = workQueue.wrapForegroundJob(function (req, res, next) {
   let method = req.method && req.method.toLowerCase()
 
   if (method !== 'get') {
@@ -275,7 +333,7 @@ Collection.prototype.versions = function (req, res, next) {
   }).catch(error => {
     return help.sendBackJSON(null, res, next)(error)
   })
-}
+})
 
 module.exports = function (model, server) {
   return new Collection(model, server)

dadi/lib/controller/index.js

@@ -157,61 +157,6 @@ Controller.prototype._prepareQueryOptions = function (options) {
 
 Controller.prototype.ID_PATTERN = ID_PATTERN
 
-/**
- * Handle collection search endpoints
- * Example: /1.0/library/books/search?q=title
- */
-Controller.prototype.search = function (req, res, next) {
-  let path = url.parse(req.url, true)
-  let options = path.query
-
-  let queryOptions = this._prepareQueryOptions(options)
-
-  if (queryOptions.errors.length !== 0) {
-    return help.sendBackJSON(400, res, next)(null, queryOptions)
-  } else {
-    queryOptions = queryOptions.queryOptions
-  }
-
-  return this.model.search({
-    client: req.dadiApiClient,
-    options: queryOptions
-  }).then(query => {
-    let ids = query._id['$containsAny'].map(id => id.toString())
-
-    return this.model.find({
-      client: req.dadiApiClient,
-      language: options.lang,
-      query,
-      options: queryOptions
-    }).then(results => {
-      results.results = results.results.sort((a, b) => {
-        let aIndex = ids.indexOf(a._id.toString())
-        let bIndex = ids.indexOf(b._id.toString())
-
-        if (aIndex === bIndex) return 0
-
-        return aIndex > bIndex ? 1 : -1
-      })
-
-      return this.model.formatForOutput(
-        results.results,
-        {
-          client: req.dadiApiClient,
-          composeOverride: queryOptions.compose,
-          language: options.lang,
-          urlFields: queryOptions.fields
-        }
-      ).then(formattedResults => {
-        results.results = formattedResults
-        return help.sendBackJSON(200, res, next)(null, results)
-      })
-    })
-  }).catch(error => {
-    return help.sendBackJSON(null, res, next)(error)
-  })
-}
-
 module.exports = function (model) {
   return new Controller(model)
 }