Add search

config.js

@@ -133,6 +133,34 @@ var conf = convict({
       env: 'DB_AUTH_NAME'
     }
   },
+  search: {
+    enabled: {
+      doc: 'If true, API responds to collection /search endpoints',
+      format: Boolean,
+      default: false
+    },
+    minQueryLength: {
+      doc: 'Minimum search string length',
+      format: Number,
+      default: 3
+    },
+    wordCollection: {
+      doc: 'The name of the datastore collection that will hold tokenized words',
+      format: String,
+      default: 'words'
+    },
+    datastore: {
+      doc: 'The datastore to use for storing and querying indexed documents',
+      format: String,
+      default: '@dadi/api-mongodb'
+    },
+    database: {
+      doc: 'The name of the database to use for storing and querying indexed documents',
+      format: String,
+      default: 'search',
+      env: 'DB_SEARCH_NAME'
+    }
+  },
   caching: {
     ttl: {
       doc: '',

config/config.test.json.sample

@@ -47,6 +47,13 @@
     "defaultBucket": "mediaStore",
     "basePath": "test/acceptance/temp-workspace/media"
   },
+  "search": {
+    "enabled": false,
+    "minQueryLength": 3,
+    "wordCollection": "words",
+    "datastore": "./../../../test/test-connector",
+    "database": "search"
+  },
   "feedback": false,
   "cors": false,
   "cluster": false

config/mongodb.test.json

@@ -34,8 +34,13 @@
       ],
       "username": "",
       "password": "",
-      "replicaSet": "",
-      "ssl": false
-    }
-  }
+  },
+  "search": {
+    "hosts": [
+      {
+        "host": "127.0.0.1",
+        "port": 27017
+      }
+    ]
+  },
 }

dadi/lib/controller/documents.js

@@ -214,7 +214,7 @@ Collection.prototype.registerRoutes = function (route, filePath) {
   })
 
   // Creating generic route.
-  this.server.app.use(`${route}/:id(${this.ID_PATTERN})?/:action(count|stats)?`, (req, res, next) => {
+  this.server.app.use(`${route}/:id(${this.ID_PATTERN})?/:action(count|search|stats)?`, (req, res, next) => {
     try {
       // Map request method to controller method.
       let method = req.params.action || (req.method && req.method.toLowerCase())
@@ -252,7 +252,7 @@ Collection.prototype.stats = function (req, res, next) {
 
 Collection.prototype.unregisterRoutes = function (route) {
   this.server.app.unuse(`${route}/config`)
-  this.server.app.unuse(`${route}/:id(${this.ID_PATTERN})?/:action(count|stats)?`)
+  this.server.app.unuse(`${route}/:id(${this.ID_PATTERN})?/:action(count|search|stats)?`)
 }
 
 module.exports = function (model, server) {

dadi/lib/controller/index.js

@@ -102,6 +102,11 @@ Controller.prototype._prepareQueryOptions = function (options) {
     )
   }
 
+  // `q` represents a search query, e.g. `?q=foo bar baz`.
+  if (options.q) {
+    queryOptions.search = options.q
+  }
+
   // Specified / default number of records to return.
   let limit = parseInt(options.count || settings.count) || 50
 
@@ -161,6 +166,61 @@ Controller.prototype._prepareQueryOptions = function (options) {
 
 Controller.prototype.ID_PATTERN = ID_PATTERN
 
+/**
+ * Handle collection search endpoints
+ * Example: /1.0/library/books/search?q=title
+ */
+Controller.prototype.search = function (req, res, next) {
+  let path = url.parse(req.url, true)
+  let options = path.query
+
+  let queryOptions = this._prepareQueryOptions(options)
+
+  if (queryOptions.errors.length !== 0) {
+    return help.sendBackJSON(400, res, next)(null, queryOptions)
+  } else {
+    queryOptions = queryOptions.queryOptions
+  }
+
+  return this.model.search({
+    client: req.dadiApiClient,
+    options: queryOptions
+  }).then(query => {
+    let ids = query._id['$containsAny'].map(id => id.toString())
+
+    return this.model.find({
+      client: req.dadiApiClient,
+      language: options.lang,
+      query,
+      options: queryOptions
+    }).then(results => {
+      results.results = results.results.sort((a, b) => {
+        let aIndex = ids.indexOf(a._id.toString())
+        let bIndex = ids.indexOf(b._id.toString())
+
+        if (aIndex === bIndex) return 0
+
+        return aIndex > bIndex ? 1 : -1
+      })
+
+      return this.model.formatForOutput(
+        results.results,
+        {
+          client: req.dadiApiClient,
+          composeOverride: queryOptions.compose,
+          language: options.lang,
+          urlFields: queryOptions.fields
+        }
+      ).then(formattedResults => {
+        results.results = formattedResults
+        return help.sendBackJSON(200, res, next)(null, results)
+      })
+    })
+  }).catch(error => {
+    return help.sendBackJSON(null, res, next)(error)
+  })
+}
+
 module.exports = function (model) {
   return new Controller(model)
 }

dadi/lib/controller/searchIndex.js

@@ -0,0 +1,44 @@
+const acl = require('./../model/acl')
+const config = require('./../../../config')
+const help = require('./../help')
+
+const SearchIndex = function (server) {
+  this.server = server
+
+  server.app.routeMethods('/api/index', {
+    post: this.post.bind(this)
+  })
+}
+
+SearchIndex.prototype.post = function (req, res, next) {
+  if (!req.dadiApiClient.clientId) {
+    return help.sendBackJSON(null, res, next)(
+      acl.createError(req.dadiApiClient)
+    )
+  }
+
+  // 404 if Search is not enabled
+  if (config.get('search.enabled') !== true) {
+    return next()
+  }
+
+  res.statusCode = 204
+  res.end(JSON.stringify({'message': 'Indexing started'}))
+
+  try {
+    Object.keys(this.server.components).forEach(key => {
+      let value = this.server.components[key]
+
+      let hasModel = Object.keys(value).includes('model') &&
+        value.model.constructor.name === 'Model'
+
+      if (hasModel) {
+        value.model.searchHandler.batchIndex()
+      }
+    })
+  } catch (err) {
+    console.log(err)
+  }
+}
+
+module.exports = server => new SearchIndex(server)

dadi/lib/index.js

@@ -30,13 +30,13 @@ var LanguagesController = require(path.join(__dirname, '/controller/languages'))
 var MediaController = require(path.join(__dirname, '/controller/media'))
 var ResourcesController = require(path.join(__dirname, '/controller/resources'))
 var RolesController = require(path.join(__dirname, '/controller/roles'))
+var SearchIndexController = require(path.join(__dirname, '/controller/searchIndex'))
 var StatusEndpointController = require(path.join(__dirname, '/controller/status'))
 var dadiBoot = require('@dadi/boot')
 var help = require(path.join(__dirname, '/help'))
 var Model = require(path.join(__dirname, '/model'))
 var mediaModel = require(path.join(__dirname, '/model/media'))
 var monitor = require(path.join(__dirname, '/monitor'))
-var search = require(path.join(__dirname, '/search'))
 
 var config = require(path.join(__dirname, '/../../config'))
 
@@ -248,9 +248,6 @@ Server.prototype.start = function (done) {
   // caching layer
   cache(this).init()
 
-  // search layer
-  search(this)
-
   // start listening
   var server = this.server = app.listen()
 
@@ -266,6 +263,7 @@ Server.prototype.start = function (done) {
   LanguagesController(this)
   ResourcesController(this)
   RolesController(this)
+  SearchIndexController(this)
 
   this.readyState = 1
 

dadi/lib/model/collections/create.js

@@ -156,6 +156,9 @@ function create ({
         results
       }
 
+      // Asynchronous search index.
+      this.searchHandler.index(returnData.results)
+
       // Run any `afterCreate` hooks.
       if (this.settings.hooks && (typeof this.settings.hooks.afterCreate === 'object')) {
         returnData.results.forEach(document => {

dadi/lib/model/index.js

@@ -7,6 +7,7 @@ const deepMerge = require('deepmerge')
 const fields = require('./../fields')
 const History = require('./history')
 const logger = require('@dadi/logger')
+const Search = require('./../search')
 const Validator = require('./validator')
 
 /**
@@ -76,6 +77,13 @@ const Model = function (name, schema, connection, settings) {
     this.compose = this.settings.compose
   }
 
+  // setup search context
+  this.searchHandler = new Search(this)
+
+  if (this.searchHandler.canUse()) {
+    this.searchHandler.init()
+  }
+
   // Add any configured indexes.
   if (this.settings.index && !Array.isArray(this.settings.index)) {
     this.settings.index = [
@@ -781,6 +789,7 @@ Model.prototype.getStats = require('./collections/getStats')
 Model.prototype.revisions = require('./collections/getRevisions') // (!) Deprecated in favour of `getRevisions`
 Model.prototype.stats = require('./collections/getStats') // (!) Deprecated in favour of `getStats`
 Model.prototype.update = require('./collections/update')
+Model.prototype.search = require('./search')
 
 module.exports = function (name, schema, connection, settings) {
   if (schema) {

dadi/lib/model/search.js

@@ -0,0 +1,47 @@
+const config = require('./../../../config')
+const debug = require('debug')('api:model:search')
+
+/**
+ * Searches for documents in the database and returns a
+ * metadata object.
+ *
+ * @param {Object} query - the search query
+ * @param {Object} options - an options object
+ * @returns {Promise<Metadata>}
+ */
+module.exports = function ({
+  client,
+  options = {}
+} = {}) {
+  let err
+
+  if (!this.searchHandler.canUse()) {
+    err = new Error('Not Implemented')
+    err.statusCode = 501
+    err.json = {
+      errors: [{
+        message: `Search is disabled or an invalid data connector has been specified.`
+      }]
+    }
+  } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
+    err = new Error('Bad Request')
+    err.statusCode = 400
+    err.json = {
+      errors: [{
+        message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
+      }]
+    }
+  }
+
+  if (err) {
+    return Promise.reject(err)
+  }
+
+  return this.validateAccess({
+    client,
+    type: 'read'
+  }).then(() => {
+    debug(options.search)
+    return this.searchHandler.find(options.search)
+  })
+}