From 59bda75af988c0cfe3793cd62ccb9c6a8fbea608 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 31 Oct 2017 18:31:11 +0000
Subject: [PATCH 01/33] feat: add search files

---
 config.js                                     |  28 +
 config/config.test.json.sample                |   7 +
 config/mongodb.test.json                      |  10 +-
 dadi/lib/controller/index.js                  |  24 +
 dadi/lib/index.js                             |  51 +-
 dadi/lib/model/index.js                       |  73 +++
 dadi/lib/search/analysers/standard.js         | 107 ++++
 dadi/lib/search/index.js                      | 591 ++++++++++++++++--
 dadi/lib/search/util.js                       |   7 +
 package.json                                  |   1 +
 test/acceptance/db-connection.js              |  27 +-
 test/acceptance/search.js                     | 186 ++++++
 .../vtest/testdb/collection.articles.json     |   6 +-
 .../vtest/testdb/collection.publications.json |   2 +-
 .../vtest/testdb/collection.test-schema.json  |  17 +-
 test/unit/search/analysers/standard.js        | 127 ++++
 test/unit/search/index.js                     | 330 ++++++++++
 test/unit/search/util.js                      |  17 +
 .../vjoin/testdb/collection.books.json        |  18 +-
 19 files changed, 1516 insertions(+), 113 deletions(-)
 create mode 100644 dadi/lib/search/analysers/standard.js
 create mode 100644 dadi/lib/search/util.js
 create mode 100644 test/acceptance/search.js
 create mode 100644 test/unit/search/analysers/standard.js
 create mode 100644 test/unit/search/index.js
 create mode 100644 test/unit/search/util.js

diff --git a/config.js b/config.js
index beae3fee..4c2d3109 100755
--- a/config.js
+++ b/config.js
@@ -133,6 +133,34 @@ var conf = convict({
       default: 3600
     }
   },
+  search: {
+    enabled: {
+      doc: 'If enabled, search will index content',
+      format: Boolean,
+      default: false
+    },
+    minQueryLength: {
+      doc: 'Minimum search string length',
+      format: Number,
+      default: 3
+    },
+    wordCollection: {
+      doc: '',
+      format: String,
+      default: 'words'
+    },
+    datastore: {
+      doc: "",
+      format: String,
+      default: '@dadi/api-mongodb'
+    },
+    database: {
+      doc: '',
+      format: String,
+      default: 'test',
+      env: 'DB_SEARCH_NAME'
+    }
+  },
   caching: {
     ttl: {
       doc: '',
diff --git a/config/config.test.json.sample b/config/config.test.json.sample
index 556a3ed9..5e24b9e0 100755
--- a/config/config.test.json.sample
+++ b/config/config.test.json.sample
@@ -48,6 +48,13 @@
     "defaultBucket": "mediaStore",
     "basePath": "test/acceptance/workspace/media"
   },
+  "search": {
+    "enabled": true,
+    "minQueryLength": 3,
+    "wordCollection": "words",
+    "datastore": "@dadi/api-mongodb",
+    "database": "search"
+  },
   "feedback": false,
   "cors": false,
   "cluster": false
diff --git a/config/mongodb.test.json b/config/mongodb.test.json
index d91a5a17..c35c235e 100644
--- a/config/mongodb.test.json
+++ b/config/mongodb.test.json
@@ -43,5 +43,13 @@
     "password": "",
     "replicaSet": "",
     "ssl": false
-  }
+  },
+  "search": {
+    "hosts": [
+      {
+        "host": "127.0.0.1",
+        "port": 27017
+      }
+    ]
+  },
 }
diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index 37357b51..607ac49f 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -59,6 +59,11 @@ function prepareQueryOptions (options, modelSettings) {
     response.errors.push(new ApiError('Bad Request', 'Invalid Parameter', 'The `page` parameter must be greater than zero', 'Invalid Page Parameter Provided'))
   }
 
+  // `q` represents a search query, e.g. `?q=foo bar baz`.
+  if (options.q) {
+    queryOptions.search = options.q
+  }
+
   // specified / default number of records to return
   let limit = options.count || settings.count
 
@@ -289,6 +294,25 @@ Controller.prototype.delete = function (req, res, next) {
   })
 }
 
+/**
+ * Handle collection search endpoints
+ * Example: /1.0/library/books/search?q=title
+ */
+Controller.prototype.search = function (req, res, next) {
+  var path = url.parse(req.url, true)
+  var options = path.query
+
+  var queryOptions = this.prepareQueryOptions(options)
+
+  if (queryOptions.errors.length !== 0) {
+    sendBackJSON(400, res, next)(null, queryOptions)
+  } else {
+    queryOptions = queryOptions.queryOptions
+  }
+
+  this.model.search(queryOptions, sendBackJSON(200, res, next), req)
+}
+
 Controller.prototype.stats = function (req, res, next) {
   this.model.stats({}, function (err, stats) {
     if (err) return next(err)
diff --git a/dadi/lib/index.js b/dadi/lib/index.js
index d6958073..4f2f7084 100755
--- a/dadi/lib/index.js
+++ b/dadi/lib/index.js
@@ -30,7 +30,6 @@ var help = require(path.join(__dirname, '/help'))
 var Model = require(path.join(__dirname, '/model'))
 var mediaModel = require(path.join(__dirname, '/model/media'))
 var monitor = require(path.join(__dirname, '/monitor'))
-var search = require(path.join(__dirname, '/search'))
 
 var config = require(path.join(__dirname, '/../../config'))
 var configPath = path.resolve(config.configPath())
@@ -224,9 +223,6 @@ Server.prototype.start = function (done) {
   // caching layer
   cache(self).init()
 
-  // search layer
-  search(self)
-
   // start listening
   var server = this.server = app.listen()
 
@@ -238,6 +234,7 @@ Server.prototype.start = function (done) {
   this.loadCollectionRoute()
   this.loadEndpointsRoute()
   this.loadHooksRoute(options)
+  this.loadSearchIndexRoute()
 
   this.readyState = 1
 
@@ -663,6 +660,40 @@ Server.prototype.loadHooksRoute = function (options) {
   })
 }
 
+Server.prototype.loadSearchIndexRoute = function () {
+  this.app.use('/api/index', (req, res, next) => {
+    const method = req.method && req.method.toLowerCase()
+
+    // 404 if Search is not enabled
+    if (config.get('search.enabled') === false) {
+      return next()
+    }
+
+    // 405 if request is not POST
+    if (method !== 'post') {
+      return help.sendBackJSON(405, res, next)(null, {'error': 'Invalid method'})
+    }
+
+    res.statusCode = 204
+    res.end(JSON.stringify({'message': 'Indexing started'}))
+
+    try {
+      Object.keys(this.components).forEach(key => {
+        const value = this.components[key]
+
+        const hasModel = Object.keys(value).includes('model') &&
+          value.model.constructor.name === 'Model'
+
+        if (hasModel) {
+          value.model.searcher.batchIndex()
+        }
+      })
+    } catch (err) {
+      console.log(err)
+    }
+  })
+}
+
 Server.prototype.updateVersions = function (versionsPath) {
   var self = this
 
@@ -984,6 +1015,18 @@ Server.prototype.addComponent = function (options) {
   this.components[options.route] = options.component
   this.docs[options.route] = options.docs
 
+  // call controller search method
+  this.app.use(options.route + '/search', function (req, res, next) {
+    var method = req.method && req.method.toLowerCase()
+
+    // call controller stats method
+    if (method === 'get') {
+      return options.component['search'](req, res, next)
+    } else {
+      next()
+    }
+  })
+
   this.app.use(options.route + '/count', function (req, res, next) {
     var method = req.method && req.method.toLowerCase()
 
diff --git a/dadi/lib/model/index.js b/dadi/lib/model/index.js
index 987ad2dd..b39d1f8b 100755
--- a/dadi/lib/model/index.js
+++ b/dadi/lib/model/index.js
@@ -12,6 +12,7 @@ const formatError = require('@dadi/format-error')
 const History = require(path.join(__dirname, '/history'))
 const Hook = require(path.join(__dirname, '/hook'))
 const logger = require('@dadi/logger')
+const Search = require(path.join(__dirname, '/../search'))
 const queryUtils = require(path.join(__dirname, '/utils'))
 const Validator = require(path.join(__dirname, '/validator'))
 
@@ -100,6 +101,17 @@ const Model = function (name, schema, conn, settings) {
     this.revisionCollection = (this.settings.revisionCollection ? this.settings.revisionCollection : this.name + 'History')
   }
 
+  // setup search context
+  this.searcher = new Search(this)
+
+  if (this.searcher.canUse()) {
+    this.searcher.init()
+  }
+
+  if (this.settings.index) {
+    this.createIndex(() => {})
+  }
+
   if (this.settings.index) {
     this.createIndex(() => {})
   }
@@ -252,6 +264,9 @@ Model.prototype.create = function (documents, internals, done, req) {
       this.composer.compose(returnData.results, (obj) => {
         returnData.results = obj
 
+        // Asynchronous search index
+        this.searcher.index(returnData.results)
+
         // apply any existing `afterCreate` hooks
         if (this.settings.hasOwnProperty('hooks') && (typeof this.settings.hooks.afterCreate === 'object')) {
           returnData.results.forEach((doc) => {
@@ -348,6 +363,9 @@ Model.prototype.delete = function (query, done, req) {
       schema: this.schema
     }).then(result => {
       if (result.deletedCount > 0) {
+        // clear documents from search index
+        this.searcher.delete(deletedDocs)
+
         // apply any existing `afterDelete` hooks
         if (this.settings.hasOwnProperty('hooks') && (typeof this.settings.hooks.afterDelete === 'object')) {
           this.settings.hooks.afterDelete.forEach((hookConfig, index) => {
@@ -687,6 +705,58 @@ Model.prototype.find = function (query, options, done) {
   return _done(this.connection.db)
 }
 
+/**
+ * Lookup documents in the database based on search query and run any associated hooks
+ *
+ * @param {Object} query
+ * @param {Function} done
+ * @return undefined
+ * @api public
+ */
+Model.prototype.search = function (options, done, req) {
+  let err
+
+  if (typeof options === 'function') {
+    done = options
+    options = {}
+  }
+
+  if (!this.searcher.canUse()) {
+    err = new Error('Not Implemented')
+    err.statusCode = 501
+    err.message = `Search is disabled or an invalid data connector has been specified`
+  } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
+    err = new Error('Bad Request')
+    err.statusCode = 400
+    err.message = `Search query must be at least ${config.get('search.minQueryLength')} characters`
+  }
+
+  if (err) {
+    return done(err, null)
+  }
+
+  this.searcher.find(options.search).then(query => {
+    const ids = query._id.$in.map(id => id.toString())
+
+    this.get(query, options, (err, results) => {
+      // sort the results
+      results.results = results.results.sort((a, b) => {
+        const aIndex = ids.indexOf(a._id.toString())
+        const bIndex = ids.indexOf(b._id.toString())
+
+        if (aIndex === bIndex) return 1
+
+        return aIndex < bIndex ? -1 : 1
+      })
+
+      return done(err, results)
+    }, req)
+  }).catch(err => {
+    console.log(err)
+    return done(err)
+  })
+}
+
 /**
  * Performs a last round of formatting to the query before it's
  * delivered to the data adapters
@@ -1118,6 +1188,9 @@ Model.prototype.update = function (query, update, internals, done, req, bypassOu
               // apply any existing `afterUpdate` hooks
               triggerAfterUpdateHook(results.results)
 
+              // asynchronous search index
+              this.searcher.index(results.results)
+
               // Prepare result set for output
               if (!bypassOutputFormatting) {
                 results.results = this.formatResultSetForOutput(results.results)
diff --git a/dadi/lib/search/analysers/standard.js b/dadi/lib/search/analysers/standard.js
new file mode 100644
index 00000000..f3412ed7
--- /dev/null
+++ b/dadi/lib/search/analysers/standard.js
@@ -0,0 +1,107 @@
+'use strict'
+const natural = require('natural')
+const tokenizer = new natural.WordTokenizer()
+const util = require('../util')
+const TfIdf = natural.TfIdf
+
+module.exports = class StandardAnalyzer {
+  constructor (fieldRules) {
+    this.fieldRules = fieldRules
+    this.tfidf = new TfIdf()
+  }
+
+  add (field, value) {
+    if (Array.isArray(value)) {
+      const filteredValues = value.filter(this.isValid)
+      filteredValues.forEach(val => this.tfidf.addDocument(val, field))
+    } else if (this.isValid(value)) {
+      this.tfidf.addDocument(value, field)
+    }
+  }
+
+  isValid (value) {
+    return typeof value === 'string'
+  }
+
+  getWordsInField (index) {
+    return this.tfidf.listTerms(index)
+      .map(item => item.term)
+  }
+
+  getAllWords () {
+    let words = this.tfidf.documents.map((doc, indx) => {
+      return this.getWordsInField(indx)
+    })
+
+    if (words.length) {
+      words = words.reduce(util.mergeArrays)
+    }
+
+    return this.unique(words)
+  }
+
+  tokenize (query) {
+    return tokenizer
+      .tokenize(query)
+      .map(word => word.toLowerCase())
+  }
+
+  unique (list) {
+    if (!Array.isArray(list)) return []
+    return [...new Set(list)]
+  }
+
+  areValidWords (words) {
+    return Array.isArray(words) &&
+    words.every(word => {
+      return typeof word === 'object' &&
+        word.hasOwnProperty('weight') &&
+        word.hasOwnProperty('word')
+    })
+  }
+
+  mergeWeights (words) {
+    if (!this.areValidWords(words)) return []
+
+    return words
+      .reduce((prev, curr) => {
+        const match = prev.find(wordSearch => wordSearch.word === curr.word)
+
+        if (match) {
+          match.count = match.count ? match.count + 1 : 2
+          match.weight += curr.weight
+          return prev
+        }
+        return prev.concat(curr)
+      }, [])
+      .map(match => {
+        if (match.count) {
+          match.weight = match.weight / match.count
+          delete match.count
+        }
+        return match
+      })
+  }
+
+  getWordInstances () {
+    const words = this.getAllWords()
+    if (!words || !words.length) return []
+
+    const docWords = this.tfidf.documents
+      .map((doc, index) => {
+        const rules = this.fieldRules[doc.__key]
+
+        return words
+          .filter(word => doc[word])
+          .map(word => {
+            const weight = this.tfidf.tfidf(word, index) * rules.weight
+            return {
+              weight,
+              word
+            }
+          })
+      }).reduce((a, b) => a.concat(b))
+
+    return this.mergeWeights(docWords)
+  }
+}
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index 0013ad43..ed85bcd7 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -1,87 +1,556 @@
-var _ = require('underscore')
-var path = require('path')
-var url = require('url')
-var help = require(path.join(__dirname, '/../help'))
-var model = require(path.join(__dirname, '/../model'))
-/*
+'use strict'
 
-Search middleware allowing cross-collection querying
+const path = require('path')
+const config = require(path.join(__dirname, '/../../../config'))
+const Connection = require(path.join(__dirname, '/../model/connection'))
+const StandardAnalyser = require('./analysers/standard')
+const DefaultAnalyser = StandardAnalyser
+const allowedDatastores = ['@dadi/api-mongodb']
+const pageLimit = 20
 
-Search query URI format:
+/**
+ * Handles collection searching in API
+ * @constructor Search
+ * @classdesc Indexes documents as they are inserted/updated, and performs search tasks.
+ * N.B. May only be used with the MongoDB Data Connector.
+ */
+const Search = function (model) {
+  if (!model || model.constructor.name !== 'Model') throw new Error('model should be an instance of Model')
 
-http://host[:port]/version/search?collections=database/collection[,database2/collection2,...[,databaseN/collectionN]]&query={"title":{"$regex":"brother"}}
+  this.model = model
+  this.indexableFields = this.getIndexableFields()
+}
 
-Example search query:
+/**
+ * Determines if searching is enabled for the current collection. Search is available
+ * if the main configuration setting of "enabled" is "true", and if the current collection
+ * schema contains at least one indexable field. An indexable field is one that has the following
+ * configuration:
+ *
+ * ```json
+ * "search": {
+ *   "weight": 2
+ * }
+ * ```
+ * @returns {Boolean} - boolean value indicating whether Search is enabled for this collection
+ */
+Search.prototype.canUse = function () {
+  const searchConfig = config.get('search')
+  const indexfieldCount = Object.keys(this.indexableFields || {}).length
 
-http://api.example.com/1.0/search?collections=library/books,library/films&query={"title":{"$regex":"brother"}}
+  let canUse = searchConfig.enabled &&
+  indexfieldCount > 0 &&
+  allowedDatastores.includes(searchConfig.datastore)
 
-*/
-module.exports = function (server) {
-  server.app.use('/:version/search', function (req, res, next) {
-    // sorry, we only process GET requests at this endpoint
-    var method = req.method && req.method.toLowerCase()
-    if (method !== 'get') {
-      return next()
-    }
+  return canUse
+}
+
+/**
+ *
+ */
+Search.prototype.init = function () {
+  this.wordCollection = config.get('search.wordCollection')
+  this.searchCollection = this.model.searchCollection || this.model.name + 'Search'
+
+  this.initialiseConnections()
+  this.applyIndexListeners()
+}
+
+/**
+ * Initialise connections to the `word` database collection and the current collection's
+ * `search` database collection - typically the collection name with "Search" appended.
+ */
+Search.prototype.initialiseConnections = function () {
+  const searchConfig = config.get('search')
+
+  this.wordConnection = Connection(
+    {
+      database: searchConfig.database,
+      collection: this.wordCollection,
+      override: true
+    },
+    this.wordCollection,
+    searchConfig.datastore
+  )
+
+  this.searchConnection = Connection(
+    {
+      database: searchConfig.database,
+      collection: this.searchCollection,
+      override: true
+    },
+    this.searchCollection,
+    searchConfig.datastore
+  )
+
+  this.wordConnection.setMaxListeners(35)
+  this.searchConnection.setMaxListeners(35)
+}
+
+/**
+ * Apply Index Listeners
+ * Fires a call to the data controllers index method with the schemas index rules.
+ */
+ // TODO: this will change with @eduardo's Connection Recovery branch
+Search.prototype.applyIndexListeners = function () {
+  this.wordConnection.once('connect', database => {
+    database.index(this.wordCollection, this.getWordSchema().settings.index)
+  })
+
+  this.searchConnection.once('connect', database => {
+    database.index(this.searchCollection, this.getSearchSchema().settings.index)
+  })
+}
+
+/**
+ * Find documents in the "words" collection matching the specified searchTerm, using the results of the query
+ * to fetch results from the current collection's search collection, ultimately leading to a set of IDs for documents
+ * that contain the searchTerm
+ *
+ * @param {String} searchTerm - the search query passed to the collection search endpoint
+ * @return {Promise} - resolves with a MongoDB query containing IDs of documents that contain the searchTerm
+ */
+Search.prototype.find = function (searchTerm) {
+  if (!this.canUse()) return {}
+
+  try {
+    const analyser = new DefaultAnalyser(this.indexableFields)
+    const tokenized = analyser.tokenize(searchTerm)
+
+    return this.getWords(tokenized).then(words => {
+      return this.getInstancesOfWords(words.results).then(instances => {
+        const ids = instances.map(instance => instance._id.document)
+        return { _id: { '$in': ids } }
+      })
+    })
+  } catch (err) {
+    console.log(err)
+  }
+}
+
+/**
+ * Removes entries in the collection's search collection that match the specified documents
+ * @param {Array} docs - an array of documents for which to remove word instances
+ * @return {Promise} - Query to delete instances with matching document ids.
+ */
+Search.prototype.delete = function (docs) {
+  if (!this.canUse()) return Promise.resolve()
 
-    var path = url.parse(req.url, true)
-    var options = path.query
+  if (!Array.isArray(docs)) return
 
-    // no collection and no query params
-    if (!(options.collections && options.query)) {
-      return help.sendBackJSON(400, res, next)(null, {'error': 'Bad Request'})
+  const deleteQueue = docs
+    .map(doc => this.clearDocumentInstances(doc._id.toString()))
+
+  return Promise.all(deleteQueue)
+}
+
+/**
+ * Query the "words" collection for results that maych any of the words specified. If there are no
+ * results, re-query the collection using the same set of words but each converted to a regular expression
+ *
+ * @param {Array} words - an array of words extracted from the search term
+ * @return {Promise} Query against the words collection.
+ */
+Search.prototype.getWords = function (words) {
+  const wordQuery = { word: { '$in': words } }
+
+  return this.runFind(
+    this.wordConnection.db,
+    wordQuery,
+    this.wordCollection,
+    this.getWordSchema().fields
+  ).then(response => {
+    // Try a second pass with regular expressions
+    if (!response.length) {
+      const regexWords = words.map(word => new RegExp(word))
+      const regexQuery = { word: { '$in': regexWords } }
+
+      return this.runFind(this.wordConnection.db, regexQuery, this.wordCollection, this.getWordSchema().fields)
     }
 
-    // split the collections param
-    var collections = options.collections.split(',')
+    return response
+  })
+}
+
+/**
+ * Searches the collection's "search" collection using the word ID to obtain document IDs for querying the main collection
+ * The "words" argument should be similar to the following exampe:
+ * ```
+ * [ { _id: 59f2e4be2b58ff41a4f9c14b, word: 'quick' }, { _id: 59f2e4be2b58ff41a4f9c14c, word: 'brown' } ]
+ * ```
+ * @param {Array} words - an array of "word" result objects, each containing an ID that can be used to query the search collection
+ * @returns {Promise.<Array, Error>} A Promise that returns an Array containing found instances of the specified words
+ * ```
+ * [
+ *   {
+ *     _id: {
+ *       document: '59f2e8fb01eaec491579ff9d'
+ *     },
+ *     count: 2,
+ *     weight: 1.2274112777602189
+ *   }
+ * ]
+ * ```
+ */
+Search.prototype.getInstancesOfWords = function (words) {
+  const ids = words.map(word => word._id.toString())
+
+  // construct an aggregation query for MongoDB
+  const query = [
+    {
+      $match: {
+        word: {
+          $in: ids
+        }
+      }
+    },
+    {
+      $group: {
+        _id: { document: '$document' },
+        count: { $sum: 1 },
+        weight: { $sum: '$weight' }
+      }
+    },
+    {
+      $sort: {
+        weight: -1
+      }
+    },
+    { $limit: pageLimit }
+  ]
+
+  return this.runFind(this.searchConnection.db, query, this.searchCollection, this.getSearchSchema().fields)
+}
+
+/**
+ * Returns all fields from the current collction's schema that have a valid search property
+ * @return {Object} - an object whose keys are the index fields, the value of which represents it's search rules
+ * ```json
+ * { title: { indexed: true, store: true, weight: 2 } }
+ * ```
+ */
+Search.prototype.getIndexableFields = function () {
+  const schema = this.model.schema
+
+  return Object.assign({}, ...Object.keys(schema)
+    .filter(key => this.hasSearchField(schema[key]))
+    .map(key => {
+      return {[key]: schema[key].search}
+    }))
+}
+
+/**
+ * Determine if the specified collection schema field has a valid search property
+ * @param {Object} field - a collection schema field object
+ * @return {Boolean} - `true` if the field has a valid search property
+ */
+Search.prototype.hasSearchField = function (field) {
+  return typeof field === 'object' &&
+    field.search &&
+    !isNaN(field.search.weight)
+}
+
+/**
+ * Removes properties from the specified document that aren't configured to be indexed
+ *
+ * @param  {Object} doc - a document to be indexed
+ * @return {Object} - the specified document with non-indexable properties removed
+ */
+Search.prototype.removeNonIndexableFields = function (doc) {
+  if (typeof doc !== 'object') return {}
+
+  return Object.assign({}, ...Object.keys(doc)
+    .filter(key => this.indexableFields[key])
+    .map(key => {
+      return {[key]: doc[key]}
+    }))
+}
+
+/**
+ * Index the specified documents
+ * @param {Array} docs - an array of documents to be indexed
+ * @return {Promise} - Queries to index documents.
+ */
+Search.prototype.index = function (docs) {
+  if (!this.canUse() || !Array.isArray(docs)) return Promise.resolve()
 
-    // extract the query from the querystring
-    var query = help.parseQuery(options.query)
+  return Promise.all(docs.map(doc => this.indexDocument(doc)))
+}
 
-    // determine API version
-    var apiVersion = path.pathname.split('/')[1]
+/**
+ * Index the specified document, inserting words from the indexable fields into the
+  * "words" collection
+ * @param {Object} doc - a document to be indexed
+ * @return {[type]}     [description]
+ */
+Search.prototype.indexDocument = function (doc) {
+  const analyser = new DefaultAnalyser(this.indexableFields)
+  const reducedDoc = this.removeNonIndexableFields(doc)
+  const words = this.analyseDocumentWords(analyser, reducedDoc)
+  const wordInsert = this.createWordInstanceInsertQuery(words)
 
-    // no collections specfied
-    if (collections.length === 0) {
-      return help.sendBackJSON(400, res, next)(null, {'error': 'Bad Request'})
+  // insert unique words into the words collection
+  return this.insert(
+    this.wordConnection.db,
+    wordInsert,
+    this.wordCollection,
+    this.getWordSchema().fields,
+    { ordered: false }
+  )
+  .then(res => {
+    return this.clearAndInsertWordInstances(words, analyser, doc._id.toString())
+  })
+  .catch(err => {
+    // code `11000` returns if the word already exists, continue regardless
+    if (err.code === 11000) {
+      return this.clearAndInsertWordInstances(words, analyser, doc._id.toString())
     }
+  })
+}
 
-    var results = {}
-    var idx = 0
+/**
+ * Analyse Document Words
+ * Pass all words to an instance of analyser and return all words.
+ * @param  {Object} doc A document from the database, with non-indexable fields removed.
+ * @return {Array} A list of analysed words.
+ */
+Search.prototype.analyseDocumentWords = function (analyserInstance, doc) {
+  // Analyse each field
+  Object.keys(doc)
+    .map(key => {
+      analyserInstance.add(key, doc[key])
+    })
 
-    _.each(collections, function (collection) {
-      // get the database and collection name from the
-      // collection parameter
-      var parts = collection.split('/')
-      var database, name, mod
+  return analyserInstance.getAllWords()
+}
 
-      query._apiVersion = apiVersion
+/**
+ * Formats the specified words for inserting into the database
+ *
+ * @param  {Array} words - an array of words
+ * @return {Array} - an array of objects in the format `{ word: <the array item> }`
+ */
+Search.prototype.createWordInstanceInsertQuery = function (words) {
+  return words.map(word => {
+    return {word}
+  })
+}
 
-      if (_.isArray(parts) && parts.length > 1) {
-        database = parts[0]
-        name = parts[1]
-        mod = model(name, null, null, database)
+/**
+ * Find all words that exist in the current version of a document, removes all indexed words relating to a specific document, and finally insert new word instances
+ * @param  {Array} words - an array of words matching document word list.
+ * @param  {Class} analyser - an analyser
+ * @param  {String} docId - the current document ID
+ * @return {Promise} Chained word query, document instance delete and document instance insert.
+ */
+Search.prototype.clearAndInsertWordInstances = function (words, analyser, docId) {
+  // The word index is unique, so results aren't always returned.
+  // Fetch word entries again to get ids.
+  const query = { word: { '$in': words } }
+
+  return this.runFind(
+    this.wordConnection.db,
+    query,
+    this.wordCollection,
+    this.getWordSchema().fields
+  ).then(results => {
+    // Get all word instances from Analyser
+    this.clearDocumentInstances(docId).then(response => {
+      if (response.deletedCount) {
+        // console.log(`Cleared ${response.deletedCount} documents`)
       }
 
-      if (mod) {
-        // query!
-        mod.find(query, function (err, docs) {
-          if (err) {
-            return help.sendBackJSON(500, res, next)(err)
-          }
+      this.insertWordInstances(analyser, results.results, docId)
+    })
+  })
+  .catch(err => {
+    console.log(err)
+  })
+}
+
+/**
+ * Insert Word Instance
+ * Insert Document word instances.
+ * @param  {Class} analyser Instance of document populated analyser class.
+ * @param  {[type]} words Results from database query for word list.
+ * @param  {String} docId Current document ID.
+ * @return {Promise} Insert query for document word instances.
+ */
+Search.prototype.insertWordInstances = function (analyser, words, docId) {
+  const instances = analyser.getWordInstances()
 
-          // add data to final results array, keyed
-          // on collection name
-          results[name] = docs
+  if (!instances) return
 
-          idx++
+  const doc = instances
+    .filter(instance => words.find(wordResult => wordResult.word === instance.word))
+    .map(instance => {
+      const word = words.find(wordResult => wordResult.word === instance.word)._id.toString()
 
-          // send back data
-          if (idx === collections.length) {
-            return help.sendBackJSON(200, res, next)(err, results)
-          }
-        })
-      }
+      return Object.assign(instance, {word, document: docId})
     })
+
+  // Insert word instances into search collection.
+  this.insert(this.searchConnection.db, doc, this.searchCollection, this.getSearchSchema().fields)
+}
+
+/**
+ * Run Find
+ * Executes find method on database
+ * @param {Connection} database Instance of database connection.
+ * @param {Object} query Query object filter.
+ * @param {String} collection Name of collection to query.
+ * @param {Object} schema Field schema for collection.
+ * @param {Object} options Query options.
+ * @return {Promise} Database find query.
+ */
+Search.prototype.runFind = function (database, query, collection, schema, options = {}) {
+  return database.find({query, collection, options, schema})
+}
+
+/**
+ * Remove entries in the collection's search collection that match the specified document ID
+ *
+ * @param  {String} docId - the document ID to remove word instances for
+ * @return {Promise} - Database delete query.
+ */
+Search.prototype.clearDocumentInstances = function (docId) {
+  const query = {document: docId}
+  return this.searchConnection.db.delete({query, collection: this.searchCollection, schema: this.getSearchSchema().fields})
+}
+
+/**
+ * Insert documents into the database
+ *
+ * @param {Connection} database - the database connection
+ * @param {Object|Array} data - the data to insert into the database
+ * @param {String} collection - the name of the collection to insert into
+ * @param {Object} schema - the collection schema
+ * @param {Object} options - options to use in the query
+ * @return {Promise}
+ */
+Search.prototype.insert = function (database, data, collection, schema, options = {}) {
+  if (!data.length) return Promise.resolve()
+  return database.insert({data, collection, options, schema})
+}
+
+/**
+ * Index an entire collection, in batches of documents
+ *
+ * @param  {Number} page - the current page of documents to process
+ * @param  {Number} limit - the number of documents to process
+ */
+Search.prototype.batchIndex = function (page = 1, limit = 1000) {
+  if (!Object.keys(this.indexableFields).length) return
+
+  const skip = (page - 1) * limit
+  console.log(`Start indexing page ${page} (${limit} per page)`)
+
+  const fields = Object.assign({}, ...Object.keys(this.indexableFields).map(key => {
+    return {[key]: 1}
+  }))
+
+  const options = {
+    skip,
+    page,
+    limit,
+    fields
+  }
+
+  if (this.model.connection.db) {
+    this.runBatchIndex(options)
+  }
+
+  this.model.connection.once('connect', database => {
+    this.runBatchIndex(options)
+  })
+}
+
+/**
+ * Run Batch Index
+ * Performs indexing across an entire collection.
+ * @param  {Object} options find query options.
+ */
+Search.prototype.runBatchIndex = function (options) {
+  this.runFind(
+    this.model.connection.db,
+    {},
+    this.model.name,
+    this.model.schema,
+    options
+  ).then(results => {
+    if (results.results && results.results.length > 0) {
+      this.index(results.results).then(response => {
+        console.log(`Indexed page ${options.page}/${results.metadata.totalPages}`)
+
+        if (options.page * options.limit < results.metadata.totalCount) {
+          return this.batchIndex(options.page + 1, options.limit)
+        } else {
+          console.log(`Indexed ${results.results.length} records for ${this.model.name}`)
+        }
+      })
+    } else {
+      console.log(`Indexed ${results.results.length} records for ${this.model.name}`)
+    }
   })
 }
+
+/**
+ * Return the template for the "words" collection schema, used to create the database collection
+ * @return {Object} - the collection schema for the "words" collection
+ */
+Search.prototype.getWordSchema = function () {
+  return {
+    fields: {
+      word: {
+        type: 'String',
+        required: true
+      }
+    },
+    settings: {
+      cache: true,
+      index: [{
+        keys: { word: 1 },
+        options: { unique: true }
+      }]
+    }
+  }
+}
+
+/**
+ * Return the template for the current collection's "search" collection schema, used to create the database collection
+ * @return {Object} - the collection schema for the "search" collection
+ */
+Search.prototype.getSearchSchema = function () {
+  return {
+    fields: {
+      word: {
+        type: 'Reference',
+        required: true
+      },
+      document: {
+        type: 'Reference',
+        required: true
+      },
+      weight: {
+        type: 'Number',
+        required: true
+      }
+    },
+    settings: {
+      cache: true,
+      index: [
+        {
+          keys: { word: 1 }
+        },
+        {
+          keys: { document: 1 }
+        },
+        {
+          keys: { weight: 1 }
+        }
+      ]
+    }
+  }
+}
+
+module.exports = Search
diff --git a/dadi/lib/search/util.js b/dadi/lib/search/util.js
new file mode 100644
index 00000000..24f551ca
--- /dev/null
+++ b/dadi/lib/search/util.js
@@ -0,0 +1,7 @@
+'use strict'
+
+const mergeArrays = (a, b) => a.concat(b)
+
+module.exports = {
+  mergeArrays
+}
diff --git a/package.json b/package.json
index 12e31a4a..f1da84d8 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
     "length-stream": "^0.1.1",
     "mkdirp": "^0.5.1",
     "moment": "2.18.1",
+    "natural": "^0.5.4",
     "parse-comments": "0.4.3",
     "path-to-regexp": "~1.7.0",
     "recovery": "^0.2.6",
diff --git a/test/acceptance/db-connection.js b/test/acceptance/db-connection.js
index 279b1cff..7af02934 100644
--- a/test/acceptance/db-connection.js
+++ b/test/acceptance/db-connection.js
@@ -88,7 +88,8 @@ describe('Database connection', () => {
 
           res.statusCode.should.eql(200)
           res.body.should.eql(mockResults)
-          datastore._spies.index.calledOnce.should.eql(true)
+
+          datastore._spies.index.callCount.should.be.above(0)
 
           setTimeout(() => {
             client
@@ -119,7 +120,7 @@ describe('Database connection', () => {
 
           res.statusCode.should.eql(200)
           res.body.should.eql(mockResults)
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           datastore._mockDisconnect()
           client
@@ -154,7 +155,7 @@ describe('Database connection', () => {
 
           res.statusCode.should.eql(200)
           res.body.should.eql(mockResults)
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           setTimeout(() => {
             client
@@ -191,7 +192,7 @@ describe('Database connection', () => {
 
           res.statusCode.should.eql(200)
           res.body.should.eql(mockResults)
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           datastore._mockDisconnect()
 
@@ -234,7 +235,7 @@ describe('Database connection', () => {
           res.statusCode.should.eql(200)
           res.body.results[0].title.should.eql('Dadi')
           res.body.results[0].published.state.should.eql(1)
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           setTimeout(() => {
             client
@@ -279,7 +280,7 @@ describe('Database connection', () => {
           res.statusCode.should.eql(200)
           res.body.results[0].title.should.eql('Dadi')
           res.body.results[0].published.state.should.eql(1)
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           datastore._mockDisconnect()
 
@@ -303,7 +304,7 @@ describe('Database connection', () => {
                 res.body.title.should.eql('Database unavailable')
 
                 done()
-              })            
+              })
           }, 1500)
         })
     })
@@ -320,7 +321,7 @@ describe('Database connection', () => {
 
           res.statusCode.should.eql(204)
           res.body.should.eql('')
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           setTimeout(() => {
             client
@@ -351,7 +352,7 @@ describe('Database connection', () => {
 
           res.statusCode.should.eql(204)
           res.body.should.eql('')
-          datastore._spies.index.calledOnce.should.eql(true)
+          datastore._spies.index.callCount.should.be.above(0)
 
           datastore._mockDisconnect()
 
@@ -456,7 +457,7 @@ describe('Database connection', () => {
               .end((err, res) => {
                 res.statusCode.should.eql(200)
                 res.body.should.eql(mockResults)
-                datastore._spies.index.calledOnce.should.eql(true)
+                datastore._spies.index.callCount.should.be.above(0)
 
                 done()
               })
@@ -537,7 +538,7 @@ describe('Database connection', () => {
               .end((err, res) => {
                 res.statusCode.should.eql(200)
                 res.body.should.eql(mockResults)
-                datastore._spies.index.calledOnce.should.eql(true)
+                datastore._spies.index.callCount.should.be.above(0)
 
                 done()
               })
@@ -631,7 +632,7 @@ describe('Database connection', () => {
                 res.statusCode.should.eql(200)
                 res.body.results[0].title.should.eql('Dadi')
                 res.body.results[0].published.state.should.eql(1)
-                datastore._spies.index.calledOnce.should.eql(true)
+                datastore._spies.index.callCount.should.be.above(0)
 
                 done()
               })
@@ -700,7 +701,7 @@ describe('Database connection', () => {
               .end((err, res) => {
                 res.statusCode.should.eql(204)
                 res.body.should.eql('')
-                datastore._spies.index.calledOnce.should.eql(true)
+                datastore._spies.index.callCount.should.be.above(0)
 
                 done()
               })
diff --git a/test/acceptance/search.js b/test/acceptance/search.js
new file mode 100644
index 00000000..7addb190
--- /dev/null
+++ b/test/acceptance/search.js
@@ -0,0 +1,186 @@
+var _ = require('underscore')
+var should = require('should')
+var sinon = require('sinon')
+var fs = require('fs')
+var path = require('path')
+var request = require('supertest')
+var EventEmitter = require('events').EventEmitter
+var connection = require(__dirname + '/../../dadi/lib/model/connection')
+var config = require(__dirname + '/../../config')
+var help = require(__dirname + '/help')
+var app = require(__dirname + '/../../dadi/lib/')
+
+// variables scoped for use throughout tests
+var bearerToken
+var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
+var lastModifiedAt = 0
+
+describe('Search', function () {
+  this.timeout(4000)
+
+  before(function (done) {
+    app.start(() => {
+      help.dropDatabase('testdb', function (err) {
+        if (err) return done(err)
+
+        help.getBearerTokenWithAccessType('admin', function (err, token) {
+          if (err) return done(err)
+
+          bearerToken = token
+
+          // add a searchable field to the schema
+          var jsSchemaString = fs.readFileSync(__dirname + '/../new-schema.json', {encoding: 'utf8'})
+          jsSchemaString = jsSchemaString.replace('newField', 'field1')
+          var schema = JSON.parse(jsSchemaString)
+
+          schema.fields.title = _.extend({}, schema.fields.newField, {
+            type: 'String',
+            required: false,
+            search: {
+              weight: 2
+            }
+          })
+
+          var client = request(connectionString)
+
+          client
+            .post('/vtest/testdb/test-schema/config')
+            .send(JSON.stringify(schema, null, 2))
+            .set('content-type', 'text/plain')
+            .set('Authorization', 'Bearer ' + bearerToken)
+            .expect('content-type', 'application/json')
+            .end((err, res) => {
+              if (err) return done(err)
+
+              // let's wait a bit
+              setTimeout(function () {
+                done()
+              }, 500)
+            })
+        })
+      })
+    })
+  })
+
+  after(function (done) {
+    config.set('search', {
+      "enabled": false
+    })
+
+    var cleanup = function (done) {
+      // try {
+      //   fs.unlinkSync(config.get('paths').collections + '/vtest/testdb/collection.test-schema.json')
+      // } catch (e) {}
+
+      done()
+    }
+
+    help.removeTestClients(() => {
+      app.stop(() => {
+        setTimeout(() => {
+          cleanup(done)
+        }, 500)
+      })
+    })
+  })
+
+  describe('Disabled', function () {
+    it('should return 501 when calling a /search endpoint', function (done) {
+      config.set('search', {
+        "enabled": false,
+        "minQueryLength": 3,
+        "datastore": "@dadi/api-mongodb",
+        "database": "search"
+      })
+
+      var client = request(connectionString)
+      client
+      .get('/vtest/testdb/test-schema/search')
+      .set('Authorization', 'Bearer ' + bearerToken)
+      .expect(501)
+      .end(done)
+    })
+  })
+
+  describe('Enabled', function () {
+    it('should return 400 when calling a /search endpoint with no query', function (done) {
+      config.set('search', {
+        "enabled": true,
+        "minQueryLength": 3,
+        "datastore": "@dadi/api-mongodb",
+        "database": "search"
+      })
+
+      var client = request(connectionString)
+      client
+      .get('/vtest/testdb/test-schema/search')
+      .set('Authorization', 'Bearer ' + bearerToken)
+      .expect(400)
+      .end(done)
+    })
+
+    it('should return 400 when calling a /search endpoint with a short query', function (done) {
+      config.set('search', {
+        "enabled": true,
+        "minQueryLength": 3,
+        "datastore": "@dadi/api-mongodb",
+        "database": "search"
+      })
+
+      var client = request(connectionString)
+      client
+      .get('/vtest/testdb/test-schema/search?q=xx')
+      .set('Authorization', 'Bearer ' + bearerToken)
+      .expect(400)
+      .end((err, res) => {
+        if (err) return done(err)
+        done()
+      })
+    })
+
+    it('should return empty results when no documents match a query', function (done) {
+      var client = request(connectionString)
+      client
+      .get('/vtest/testdb/test-schema/search?q=xxx')
+      .set('Authorization', 'Bearer ' + bearerToken)
+      .expect(200)
+      .end((err, res) => {
+        if (err) return done(err)
+        should.exist(res.body.results)
+        res.body.results.should.be.Array
+        res.body.results.length.should.eql(0)
+        done()
+      })
+    })
+
+    it('should return results when documents match a query', function (done) {
+      var client = request(connectionString)
+
+      var doc = {
+        field1: 'The quick brown fox jumps',
+        title: 'The quick brown fox jumps over the lazy dog'
+      }
+
+      client
+      .post('/vtest/testdb/test-schema')
+      .set('Authorization', 'Bearer ' + bearerToken)
+      .set('content-type', 'application/json')
+      .send(doc)
+      .expect(200)
+      .end((err, res) => {
+
+        client
+        .get('/vtest/testdb/test-schema/search?q=quick%20brown')
+        .set('Authorization', 'Bearer ' + bearerToken)
+        .expect(200)
+        .end((err, res) => {
+          if (err) return done(err)
+          should.exist(res.body.results)
+          res.body.results.should.be.Array
+          res.body.results.length.should.eql(1)
+          done()
+        })
+      })
+    })
+  })
+})
diff --git a/test/acceptance/workspace/collections/vtest/testdb/collection.articles.json b/test/acceptance/workspace/collections/vtest/testdb/collection.articles.json
index bb9f5c64..9532ac23 100644
--- a/test/acceptance/workspace/collections/vtest/testdb/collection.articles.json
+++ b/test/acceptance/workspace/collections/vtest/testdb/collection.articles.json
@@ -178,12 +178,12 @@
     "allowDelete": true,
     "count": 20,
     "sortOrder": 1,
-    "sort": "publicationDate",
     "index": {
       "enabled": true,
       "keys": {
         "_id": 1,
-        "urls": 1
+        "urls": 1,
+        "publicationDate": 1
       }
     },
     "hooks": {
@@ -211,6 +211,6 @@
       ],
       "afterUpdate": []
     },
-    "lastModifiedAt": 1472947876485
+    "lastModifiedAt": 1509474495923
   }
 }
\ No newline at end of file
diff --git a/test/acceptance/workspace/collections/vtest/testdb/collection.publications.json b/test/acceptance/workspace/collections/vtest/testdb/collection.publications.json
index 6f0e959b..b2a1e35a 100644
--- a/test/acceptance/workspace/collections/vtest/testdb/collection.publications.json
+++ b/test/acceptance/workspace/collections/vtest/testdb/collection.publications.json
@@ -93,6 +93,6 @@
         }
       ]
     },
-    "lastModifiedAt": 1484287084405
+    "lastModifiedAt": 1509474495854
   }
 }
\ No newline at end of file
diff --git a/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json b/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
index a4c2748a..66de1621 100755
--- a/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
+++ b/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
@@ -4,21 +4,8 @@
       "type": "String",
       "label": "Title",
       "comments": "The title of the entry",
-      "placement": "Main content",
       "validation": {},
-      "required": false,
-      "message": "",
-      "display": {
-        "index": true,
-        "edit": true
-      }
-    },
-    "title": {
-      "type": "String",
-      "required": false,
-      "search": {
-        "weight": 2
-      }
+      "required": false
     }
   },
   "settings": {
@@ -29,6 +16,6 @@
     "sortOrder": 1,
     "storeRevisions": true,
     "revisionCollection": "testSchemaHistory",
-    "lastModifiedAt": 1509360344054
+    "lastModifiedAt": 1509474499125
   }
 }
\ No newline at end of file
diff --git a/test/unit/search/analysers/standard.js b/test/unit/search/analysers/standard.js
new file mode 100644
index 00000000..4dc06913
--- /dev/null
+++ b/test/unit/search/analysers/standard.js
@@ -0,0 +1,127 @@
+const should = require('should')
+const sinon = require('sinon')
+const standardAnalyser = require(__dirname + '/../../../../dadi/lib/search/analysers/standard')
+const model = require(__dirname + '/../../../../dadi/lib/model')
+
+const indexableFields = {
+  foo: {
+    search: {
+      weight: 2
+    }
+  }
+}
+
+let analyser
+
+describe('Standard Search Analyser', () => {
+  beforeEach(done => {
+    analyser = new standardAnalyser(indexableFields)
+    done()
+  })
+
+  it('should export constructor', done => {
+    standardAnalyser.should.be.Function
+    done()
+  })
+
+  describe('`isValid` method', () => {
+    it('should return false if value is not a valid string', done => {
+      analyser.isValid(undefined).should.be.false
+      done()
+    })
+
+    it('should return true if value is a valid string', done => {
+      analyser.isValid('foo').should.be.true
+      done()
+    })
+  })
+
+  describe('`tokenize` method', () => {
+    it('should return a tokenized array of words from a string', done => {
+      const tokens = analyser.tokenize('Foo Bar Baz')
+
+      tokens.should.be.an.instanceOf(Array)
+        .and.have.lengthOf(3)
+      tokens.should.eql(['foo', 'bar', 'baz'])
+      done()
+    })
+  })
+
+  describe('`unique` method', () => {
+    it('should reduce an array to unique values', done => {
+      analyser.unique(['foo', 'foo', 'bar'])
+        .should.be.an.instanceOf(Array)
+        .and.have.lengthOf(2)
+      done()
+    })
+
+    it('should return empty array if the input is not a valid array', done => {
+      analyser.unique(undefined)
+        .should.be.an.instanceOf(Array)
+        .and.have.lengthOf(0)
+      done()
+    })
+  })
+
+  describe('`areValidWords` method', () => {
+    it('should return false if array of words is invalid', done => {
+      analyser.areValidWords(undefined)
+        .should.be.false
+
+      analyser.areValidWords([
+        {
+          word: 'foo'
+        }
+      ])
+        .should.be.false
+
+      analyser.areValidWords([
+        {
+          word: 'foo',
+          weight: 2
+        }
+      ])
+        .should.be.true
+
+      done()
+    })
+  })
+
+  describe('`mergeWeights` method', () => {
+    it('should return empty array if words are invalid', done => {
+      analyser.mergeWeights(undefined)
+        .should.be.an.instanceOf(Array)
+        .and.have.lengthOf(0)
+
+      analyser.mergeWeights([{
+        word: 'foo'
+      }])
+        .should.be.an.instanceOf(Array)
+        .and.have.lengthOf(0)
+      done()
+    })
+
+    it('should reduce multiple word instances to a unique instance of the highest weight value', done => {
+      analyser.mergeWeights([
+        { weight: 2.85116730682758, word: 'foo' },
+        { weight: 2.280933845462064, word: 'foo' }
+      ])
+      .should.be.an.instanceOf(Array)
+      .and.have.lengthOf(1)
+
+      analyser.mergeWeights([
+        { weight: 2.85116730682758, word: 'foo' },
+        { weight: 2.280933845462064, word: 'foo' }
+      ])[0]
+        .should.be.an.instanceOf(Object)
+        .and.have.property('weight', 2.5660505761448222)
+
+      done()
+    })
+  })
+})
+
+// add
+// getWordsInField
+// getAllWords
+// getWordInstances
diff --git a/test/unit/search/index.js b/test/unit/search/index.js
new file mode 100644
index 00000000..ce328942
--- /dev/null
+++ b/test/unit/search/index.js
@@ -0,0 +1,330 @@
+const config = require(__dirname + '/../../../config')
+const help = require(__dirname + '/../help')
+const Model = require(__dirname + '/../../../dadi/lib/model')
+const Search = require(__dirname + '/../../../dadi/lib/search')
+const should = require('should')
+const sinon = require('sinon')
+const store = require(config.get('search.datastore'))
+
+let mod
+let searchInstance
+
+describe('Search', () => {
+  beforeEach(done => {
+    mod = Model('testSearchModel', help.getSearchModelSchema(), null, { database: 'testdb' })
+    searchInstance = new Search(mod)
+    searchInstance.init()
+    done()
+  })
+
+  it('should export constructor', done => {
+    Search.should.be.Function
+    done()
+  })
+
+  it('should export a function that returns an instance', done => {
+      searchInstance.should.be.an.instanceOf(Search)
+      done()
+  })
+
+  it('should throw an error if model is incorrect type', done => {
+    should.throws(function () { var x = new Search() })
+    done()
+  })
+
+  describe('`initialiseConnections` method', () => {
+    it('should initialise required connections', done => {
+      searchInstance.initialiseConnections()
+
+      setTimeout(() => {
+        should.exist(searchInstance.wordConnection.db)
+        should.exist(searchInstance.searchConnection.db)
+        searchInstance.wordConnection.db.config.hosts[0].host.should.eql('127.0.0.1')
+        searchInstance.wordConnection.db.config.hosts[0].port.should.eql(27017)
+        searchInstance.searchConnection.db.config.hosts[0].host.should.eql('127.0.0.1')
+        searchInstance.searchConnection.db.config.hosts[0].port.should.eql(27017)
+
+        done()
+      }, 500)
+    })
+  })
+
+  describe('`applyIndexListeners` method', () => {
+    it('should call database index method once connection is established', done => {
+      mod = Model('testModelNew', help.getSearchModelSchema(), null, { database: 'testdb' })
+      const dbIndexStub = sinon.spy(store.prototype, 'index')
+
+      searchInstance = new Search(mod)
+
+      setTimeout(() => {
+        dbIndexStub.called.should.be.true
+        dbIndexStub.lastCall.args[0].should.eql('testModelNewSearch')
+        dbIndexStub.lastCall.args[1].should.be.Object
+        dbIndexStub.restore()
+
+        done()
+      }, 1000)
+    })
+  })
+
+  describe('`getWordSchema` method', () => {
+    it('should return an object', done => {
+      const schema = searchInstance.getWordSchema()
+      schema.should.be.Object
+      done()
+    })
+  })
+
+  describe('`getSearchSchema` method', () => {
+    it('should return an object', done => {
+      const schema = searchInstance.getSearchSchema()
+      schema.should.be.Object
+      done()
+    })
+  })
+
+  describe('`getIndexableFields` method', () => {
+    it('should return an object', done => {
+      searchInstance.getIndexableFields().should.be.Object
+      done()
+    })
+
+    it('should return an object containing only indexable fields', done => {
+      searchInstance.getIndexableFields().should.be.an.instanceOf(Object).and.have.property('searchableFieldName', {weight: 2})
+      searchInstance.getIndexableFields().should.not.have.property('fieldName')
+      searchInstance.getIndexableFields().should.not.have.property('invalidSearchableFieldName')
+      done()
+    })
+  })
+
+  describe('`removeNonIndexableFields` method', () => {
+    it('should return an object if doc is invalid', done => {
+      searchInstance.removeNonIndexableFields().should.be.Object
+      done()
+    })
+
+    it('should remove non-indexable fields from document', done => {
+      searchInstance.removeNonIndexableFields(help.getSampleSearchDocument())
+      .should.not.have.property('fieldName')
+      searchInstance.removeNonIndexableFields(help.getSampleSearchDocument())
+      .should.not.have.property('invalidSearchableFieldName')
+      searchInstance.removeNonIndexableFields(help.getSampleSearchDocument())
+      .should.have.property('searchableFieldName', 'baz')
+      done()
+    })
+  })
+
+  describe('`createWordInstanceInsertQuery` method', () => {
+    it('should convert list of words to valid insert query object', done => {
+      searchInstance.createWordInstanceInsertQuery(['foo']).should.be.an.instanceOf(Array)
+      searchInstance.createWordInstanceInsertQuery(['foo'])[0].should.have.property('word', 'foo')
+      done()
+    })
+  })
+
+  describe('`hasSeachField` method', () => {
+    it('should return false if a field is invalid', done => {
+      searchInstance.hasSearchField().should.be.false
+      done()
+    })
+
+    it('should return false if a field does not contain a valid search parameter', done => {
+      searchInstance.hasSearchField({search: 'foo'}).should.be.false
+      done()
+    })
+
+    it('should return true if a field has a valid search and search weight parameter', done => {
+      searchInstance.hasSearchField({search: {weight: 2}}).should.be.true
+      done()
+    })
+  })
+
+  describe('`runFind` method', () => {
+    it('should search the database based on the query', done => {
+      const dbFindStub = sinon.spy(store.prototype, 'find')
+
+      searchInstance.runFind(searchInstance.model.connection.db, {foo: 'bar'}, searchInstance.model.name, searchInstance.model.schema, {})
+      dbFindStub.called.should.be.true
+      dbFindStub.lastCall.args[0].should.have.property('query', {foo: 'bar'})
+      dbFindStub.restore()
+
+      done()
+    })
+  })
+
+  describe('`clearDocumentInstances` method', () => {
+    it('should delete all search instance documents with filtered query', done => {
+      const dbDeleteStub = sinon.spy(store.prototype, 'delete')
+
+      searchInstance.clearDocumentInstances('mockDocId')
+      dbDeleteStub.called.should.be.true
+      dbDeleteStub.lastCall.args[0].should.have.property('query', {document: 'mockDocId'})
+      dbDeleteStub.restore()
+
+      done()
+    })
+  })
+
+  describe('`delete` method', () => {
+    it('should return without firing clearDocumentInstances if an array of documents is not provided', done => {
+      const dbDeleteStub = sinon.spy(searchInstance, 'clearDocumentInstances')
+
+      searchInstance.delete({_id: 'mockDocId'})
+      dbDeleteStub.called.should.be.false
+      dbDeleteStub.restore()
+
+      done()
+    })
+
+    it('should execute clearDocumentInstances if an array of documents is provided', done => {
+      const dbDeleteStub = sinon.spy(searchInstance, 'clearDocumentInstances')
+
+      searchInstance.delete([{_id: 'mockDocId'}])
+      dbDeleteStub.called.should.be.true
+      dbDeleteStub.lastCall.args[0].should.eql('mockDocId')
+      dbDeleteStub.restore()
+
+      done()
+    })
+  })
+
+  describe('`insert` method', () => {
+    it('should not execute the database insert if no data is provided', done => {
+      const dbInsertStub = sinon.spy(store.prototype, 'insert')
+
+      searchInstance.insert({}, {}, {}, {}, {})
+      dbInsertStub.called.should.be.false
+      dbInsertStub.restore()
+
+      done()
+    })
+  })
+
+  describe('`batchIndex` method', () => {
+    it('should not execute the runBatchIndex method if no fields can be indexed', done => {
+      let schema = help.getSearchModelSchema()
+      delete schema.searchableFieldName
+
+      let mod = Model('testSearchModel', schema, null, { database: 'testdb' })
+      const unIndexable = new Search(mod)
+      unIndexable.init()
+
+      const stub = sinon.spy(unIndexable, 'runBatchIndex')
+
+      unIndexable.batchIndex(1, 100)
+      stub.called.should.be.false
+      stub.restore()
+      done()
+    })
+
+    it('should call the runBatchIndex method with correct arguments when using defaults', done => {
+      let schema = help.getSearchModelSchema()
+      let mod = Model('testSearchModel', schema, null, { database: 'testdb' })
+      const indexable = new Search(mod)
+      indexable.init()
+
+      const stub = sinon.spy(indexable, 'runBatchIndex')
+
+      indexable.batchIndex()
+      stub.called.should.be.true
+      let args = stub.lastCall.args[0]
+      args.page.should.eql(1)
+      args.limit.should.eql(1000)
+      args.skip.should.eql(0)
+      args.fields.should.eql({searchableFieldName: 1})
+      stub.restore()
+      done()
+    })
+
+    it('should call the runBatchIndex method with correct arguments when using specific params', done => {
+      let schema = help.getSearchModelSchema()
+      let mod = Model('testSearchModel', schema, null, { database: 'testdb' })
+      const indexable = new Search(mod)
+      indexable.init()
+
+      const stub = sinon.spy(indexable, 'runBatchIndex')
+
+      indexable.batchIndex(2, 500)
+      stub.called.should.be.true
+      let args = stub.lastCall.args[0]
+      args.page.should.eql(2)
+      args.limit.should.eql(500)
+      args.skip.should.eql(500)
+      args.fields.should.eql({searchableFieldName: 1})
+      stub.restore()
+      done()
+    })
+  })
+
+  describe.skip('batchIndex', function () {
+    it('should call runBatchIndex repeatedly when there are more results', done => {
+      let schema = help.getSearchModelSchema()
+      let mod = Model('testSearchModel', schema, null, { database: 'testdb' })
+      const indexable = new Search(mod)
+      indexable.init()
+
+      const spy = sinon.spy(indexable, 'runBatchIndex')
+
+      function guid () {
+        return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+          var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8)
+          return v.toString(16)
+        })
+      }
+
+      var docs = [
+        { searchableFieldName: guid() },
+        { searchableFieldName: guid() },
+        { searchableFieldName: guid() },
+        { searchableFieldName: guid() },
+        { searchableFieldName: guid() }
+      ]
+
+      const indexStub = sinon.stub(indexable, 'index').callsFake(() => {
+        return Promise.resolve({
+          results: docs,
+          metadata: {
+            totalPages: 5,
+            totalCount: 5
+          }
+        })
+      })
+
+      mod.create(docs, {}, obj => {
+        indexable.batchIndex()
+
+        setTimeout(() => {
+          spy.restore()
+          indexStub.restore()
+          spy.callCount.should.be.above(1)
+          let args = spy.args
+          args[0][0].skip.should.eql(0)
+          args[0][0].page.should.eql(1)
+          args[1][0].skip.should.eql(1)
+          args[1][0].page.should.eql(2)
+          args[2][0].skip.should.eql(2)
+          args[2][0].page.should.eql(3)
+          args[3][0].skip.should.eql(3)
+          args[3][0].page.should.eql(4)
+          args[4][0].skip.should.eql(4)
+          args[4][0].page.should.eql(5)
+          done()
+        }, 3000)
+      })
+    })
+  })
+})
+
+// TODO: test the following
+// find
+// getWords
+// getInstancesOfWords
+// index
+// indexDocument
+// analyseDocumentWords
+// clearAndInsertWordInstances
+// insertWordInstances
+// insert
+// batchIndex
+// runBatchIndex
+// canUse
diff --git a/test/unit/search/util.js b/test/unit/search/util.js
new file mode 100644
index 00000000..09debdf3
--- /dev/null
+++ b/test/unit/search/util.js
@@ -0,0 +1,17 @@
+const should = require('should')
+const searchUtil = require(__dirname + '/../../../dadi/lib/search/util')
+
+describe('Utils', () => {
+  it('should export a function', done => {
+    searchUtil.mergeArrays.should.be.Function
+    done()
+  })
+
+  describe('`mergeArrays` method', () => {
+    it('should merge two arrays together', done => {
+      const testArray = [['foo', 'bar'], ['baz', 'qux']]
+      testArray.reduce(searchUtil.mergeArrays).should.eql(['foo', 'bar', 'baz', 'qux'])
+      done()
+    })
+  })
+})
diff --git a/workspace/collections/vjoin/testdb/collection.books.json b/workspace/collections/vjoin/testdb/collection.books.json
index cb60670d..5d9299c5 100755
--- a/workspace/collections/vjoin/testdb/collection.books.json
+++ b/workspace/collections/vjoin/testdb/collection.books.json
@@ -5,12 +5,9 @@
       "label": "name",
       "example": "War and Peace",
       "comments": "This is the book's name",
-      "placement": "Main content",
       "required": true,
-      "message": "",
-      "display": {
-        "index": true,
-        "edit": false
+      "search": {
+        "weight": 2
       }
     },
     "authorId": {
@@ -18,26 +15,17 @@
       "label": "author",
       "example": "b8b285ae-53d1-47a5-9e69-ec04",
       "comments": "This is the _id of the book's author",
-      "placement": "Main content",
       "validation": {
         "regex": {
           "pattern": "^[0-9a-fA-F]{24}$"
         }
       },
-      "required": true,
-      "message": "",
-      "display": {
-        "index": true,
-        "edit": true
-      }
+      "required": true
     }
   },
   "settings": {
     "cache": true,
     "authenticate": true,
-    "callback": null,
-    "defaultFilters": null,
-    "fieldLimiters": null,
     "count": 40,
     "sort": "name",
     "sortOrder": 1,

From 1151d1b5b34914464de7eed92a8f193d8ebf1bdd Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 31 Oct 2017 19:24:31 +0000
Subject: [PATCH 02/33] tests: skip old search methods

---
 test/acceptance/search_collections.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/acceptance/search_collections.js b/test/acceptance/search_collections.js
index ad61aab3..a11fbccb 100644
--- a/test/acceptance/search_collections.js
+++ b/test/acceptance/search_collections.js
@@ -11,7 +11,7 @@ var app = require(__dirname + '/../../dadi/lib/')
 var bearerToken
 var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
 
-describe('Search', function () {
+describe.skip('Search', function () {
   this.timeout(5000)
 
   describe('Collections', function () {

From 2eddb700cdbbf2d19d3e9e8efaef01579966508c Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Nov 2017 07:29:46 +0000
Subject: [PATCH 03/33] tests: add missing helper method

---
 test/unit/help.js | 43 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/test/unit/help.js b/test/unit/help.js
index b34a3bfd..98e2bfa3 100755
--- a/test/unit/help.js
+++ b/test/unit/help.js
@@ -10,18 +10,51 @@ module.exports.getModelSchema = function () {
       'type': 'String',
       'label': 'Title',
       'comments': 'The title of the entry',
-      'placement': 'Main content',
       'validation': {},
       'required': false,
-      'message': '',
-      'display': {
-        'index': true,
-        'edit': true
+      'message': ''
+    }
+  }
+}
+
+module.exports.getSearchModelSchema = function () {
+  return {
+    'fieldName': {
+      'type': 'String',
+      'label': 'Title',
+      'comments': 'The title of the entry',
+      'validation': {},
+      'required': false
+    },
+    'invalidSearchableFieldName': {
+      'type': 'String',
+      'label': 'Title',
+      'comments': 'The title of the entry',
+      'validation': {},
+      'required': false,
+      'search': true
+    },
+    'searchableFieldName': {
+      'type': 'String',
+      'label': 'Title',
+      'comments': 'The title of the entry',
+      'validation': {},
+      'required': false,
+      'search': {
+        'weight': 2
       }
     }
   }
 }
 
+module.exports.getSampleSearchDocument = () => {
+  return {
+    fieldName: 'foo',
+    invalidSearchableFieldName: 'bar',
+    searchableFieldName: 'baz'
+  }
+}
+
 module.exports.getModelSettings = function () {
   return {
     cache: true,

From 162a596e8e555e3a4d67335e537a5ada91854073 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Nov 2017 07:37:55 +0000
Subject: [PATCH 04/33] tests: remove old search test file

---
 test/acceptance/search_collections.js | 116 --------------------------
 1 file changed, 116 deletions(-)
 delete mode 100644 test/acceptance/search_collections.js

diff --git a/test/acceptance/search_collections.js b/test/acceptance/search_collections.js
deleted file mode 100644
index a11fbccb..00000000
--- a/test/acceptance/search_collections.js
+++ /dev/null
@@ -1,116 +0,0 @@
-var should = require('should')
-var fs = require('fs')
-var path = require('path')
-var request = require('supertest')
-var _ = require('underscore')
-var config = require(__dirname + '/../../config')
-var help = require(__dirname + '/help')
-var app = require(__dirname + '/../../dadi/lib/')
-
-// variables scoped for use throughout tests
-var bearerToken
-var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
-
-describe.skip('Search', function () {
-  this.timeout(5000)
-
-  describe('Collections', function () {
-    beforeEach(function (done) {
-      help.dropDatabase('testdb', function (err) {
-        if (err) return done(err)
-
-        app.start(function () {
-          help.getBearerTokenWithAccessType('admin', function (err, token) {
-            if (err) return done(err)
-
-            bearerToken = token
-
-              // add a new field to the schema
-            var jsSchemaString = fs.readFileSync(__dirname + '/../new-schema.json', {encoding: 'utf8'})
-            jsSchemaString = jsSchemaString.replace('newField', 'field1')
-            var schema = JSON.parse(jsSchemaString)
-
-            schema.fields.field2 = _.extend({}, schema.fields.newField, {
-              type: 'Number',
-              required: true,
-              message: 'Provide a value here, please!'
-            })
-
-            schema.fields.field3 = _.extend({}, schema.fields.newField, {
-              type: 'ObjectID',
-              required: false
-            })
-
-            schema.settings.displayName = 'Test Collection'
-            schema.settings.description = 'Test Collection'
-
-            var client = request(connectionString)
-
-            client
-              .post('/vtest/testdb/test-schema/config')
-              .send(JSON.stringify(schema, null, 2))
-              .set('content-type', 'text/plain')
-              .set('Authorization', 'Bearer ' + bearerToken)
-              .expect(200)
-              .expect('content-type', 'application/json')
-              .end(function (err, res) {
-                if (err) return done(err)
-
-                setTimeout(function() {
-                  done()
-                }, 1000)
-              })
-          })
-        })
-      })
-    })
-
-    after(function (done) {
-        // reset the schema
-      var jsSchemaString = fs.readFileSync(__dirname + '/../new-schema.json', {encoding: 'utf8'})
-      jsSchemaString = jsSchemaString.replace('newField', 'field1')
-      var schema = JSON.parse(jsSchemaString)
-
-      var client = request(connectionString)
-
-      client
-        .post('/vtest/testdb/test-schema/config')
-        .send(JSON.stringify(schema, null, 2))
-        .set('content-type', 'text/plain')
-        .set('Authorization', 'Bearer ' + bearerToken)
-        .expect(200)
-        .expect('content-type', 'application/json')
-        .end(function (err, res) {
-          if (err) return done(err)
-
-          app.stop(done)
-        })
-    })
-
-    it('should return docs from specified collections', function (done) {
-      // sample URL "/:version/search?collections=collection/model&query={"field1":{"$regex":"est"}}"
-
-      var doc = { field1: 'Test', field2: 1234 }
-
-      help.createDocWithSpecificVersion(bearerToken, 'vtest', doc, function (err, doc) {
-        if (err) return done(err)
-
-        var client = request(connectionString)
-
-        client
-          .get('/vtest/search?collections=testdb/test-schema&query={"field1":{"$regex":"est"}}')
-          .set('Authorization', 'Bearer ' + bearerToken)
-          .expect(200)
-          .expect('content-type', 'application/json')
-          .end(function (err, res) {
-            if (err) return done(err)
-            should.exist(res.body['test-schema'].results)
-            res.body['test-schema'].results.should.be.Array
-            res.body['test-schema'].results.length.should.equal(1)
-            res.body['test-schema'].results[0].field1.should.equal('Test')
-            done()
-          })
-      })
-    })
-  })
-})

From 61fce48a719e8a3a69bd380fe6a5df9e259b8b64 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Nov 2017 07:52:44 +0000
Subject: [PATCH 05/33] tests: complete batchIndex tests

---
 dadi/lib/search/index.js  | 26 +++++++++++++-------------
 test/unit/search/index.js | 25 +++++++++----------------
 2 files changed, 22 insertions(+), 29 deletions(-)

diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index ed85bcd7..eacc4634 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -443,7 +443,7 @@ Search.prototype.batchIndex = function (page = 1, limit = 1000) {
   if (!Object.keys(this.indexableFields).length) return
 
   const skip = (page - 1) * limit
-  console.log(`Start indexing page ${page} (${limit} per page)`)
+  console.log(`Indexing page ${page} (${limit} per page)`)
 
   const fields = Object.assign({}, ...Object.keys(this.indexableFields).map(key => {
     return {[key]: 1}
@@ -478,18 +478,18 @@ Search.prototype.runBatchIndex = function (options) {
     this.model.schema,
     options
   ).then(results => {
-    if (results.results && results.results.length > 0) {
-      this.index(results.results).then(response => {
-        console.log(`Indexed page ${options.page}/${results.metadata.totalPages}`)
-
-        if (options.page * options.limit < results.metadata.totalCount) {
-          return this.batchIndex(options.page + 1, options.limit)
-        } else {
-          console.log(`Indexed ${results.results.length} records for ${this.model.name}`)
-        }
-      })
-    } else {
-      console.log(`Indexed ${results.results.length} records for ${this.model.name}`)
+    if (results.results && results.results.length) {
+      console.log(`Indexed ${results.results.length} ${results.results.length === 1 ? 'record' : 'records'} for ${this.model.name}`)
+
+      if (results.results.length > 0) {
+        this.index(results.results).then(response => {
+          console.log(`Indexed page ${options.page}/${results.metadata.totalPages}`)
+
+          if (options.page * options.limit < results.metadata.totalCount) {
+            return this.batchIndex(options.page + 1, options.limit)
+          }
+        })
+      }
     }
   })
 }
diff --git a/test/unit/search/index.js b/test/unit/search/index.js
index ce328942..e9d980ce 100644
--- a/test/unit/search/index.js
+++ b/test/unit/search/index.js
@@ -1,3 +1,4 @@
+const acceptanceHelper = require(__dirname + '/../../acceptance/help')
 const config = require(__dirname + '/../../../config')
 const help = require(__dirname + '/../help')
 const Model = require(__dirname + '/../../../dadi/lib/model')
@@ -256,7 +257,13 @@ describe('Search', () => {
     })
   })
 
-  describe.skip('batchIndex', function () {
+  describe('batchIndex', function () {
+    beforeEach((done) => {
+      acceptanceHelper.dropDatabase('testdb', err => {
+        done()
+      })
+    })
+
     it('should call runBatchIndex repeatedly when there are more results', done => {
       let schema = help.getSearchModelSchema()
       let mod = Model('testSearchModel', schema, null, { database: 'testdb' })
@@ -291,7 +298,7 @@ describe('Search', () => {
       })
 
       mod.create(docs, {}, obj => {
-        indexable.batchIndex()
+        indexable.batchIndex(1, 1)
 
         setTimeout(() => {
           spy.restore()
@@ -314,17 +321,3 @@ describe('Search', () => {
     })
   })
 })
-
-// TODO: test the following
-// find
-// getWords
-// getInstancesOfWords
-// index
-// indexDocument
-// analyseDocumentWords
-// clearAndInsertWordInstances
-// insertWordInstances
-// insert
-// batchIndex
-// runBatchIndex
-// canUse

From 8cd10a307884e7aca5828a7ab5e4bacd6f60a774 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Fri, 8 Jun 2018 11:37:14 +1000
Subject: [PATCH 06/33] refactor: make search ready for future API changes

---
 dadi/lib/controller/search.js         |  92 ++++++++
 dadi/lib/index.js                     |  12 +-
 dadi/lib/model/index.js               | 108 +++++----
 dadi/lib/search/analysers/standard.js |  35 +--
 dadi/lib/search/index.js              | 319 ++++++++++++++------------
 dadi/lib/search/util.js               |   7 -
 6 files changed, 356 insertions(+), 217 deletions(-)
 create mode 100755 dadi/lib/controller/search.js
 delete mode 100644 dadi/lib/search/util.js

diff --git a/dadi/lib/controller/search.js b/dadi/lib/controller/search.js
new file mode 100755
index 00000000..e6edd940
--- /dev/null
+++ b/dadi/lib/controller/search.js
@@ -0,0 +1,92 @@
+'use strict'
+
+const path = require('path')
+const url = require('url')
+const config = require(path.join(__dirname, '/../../../config'))
+const help = require(path.join(__dirname, '/../help'))
+
+const prepareQueryOptions = require(path.join(__dirname, './index')).prepareQueryOptions
+
+const SearchController = function (model) {
+  this.model = model
+}
+
+/**
+ *
+ */
+SearchController.prototype.get = function (req, res, next) {
+  let path = url.parse(req.url, true)
+  let options = path.query
+
+  let queryOptions = prepareQueryOptions(options, this.model.settings)
+
+  if (queryOptions.errors.length !== 0) {
+    sendBackJSON(400, res, next)(null, queryOptions)
+  } else {
+    queryOptions = queryOptions.queryOptions
+  }
+
+  let err
+
+  // if (typeof options === 'function') {
+  //   done = options
+  //   options = {}
+  // }
+
+  console.log(this.model)
+  console.log(queryOptions)
+
+  if (!this.model.searchHandler.canUse()) {
+    err = new Error('Not Implemented')
+    err.statusCode = 501
+    err.json = {
+      errors: [{
+        message: `Search is disabled or an invalid data connector has been specified.`
+      }]
+    }
+  }
+
+  if (!queryOptions.search || queryOptions.search.length < config.get('search.minQueryLength')) {
+    err = new Error('Bad Request')
+    err.statusCode = 400
+    err.json = {
+      errors: [{
+        message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
+      }]
+    }
+  }
+
+  if (err) {
+    return help.sendBackJSON(400, res, next)(err)
+  }
+
+  this.model.searchHandler.find(queryOptions.search).then(query => {
+    let ids = query._id['$containsAny'].map(id => id.toString())
+
+    this.model.get(query, queryOptions, (err, results) => {
+      console.log(err, results)
+      // sort the results
+      results.results = results.results.sort((a, b) => {
+        let aIndex = ids.indexOf(a._id.toString())
+        let bIndex = ids.indexOf(b._id.toString())
+
+        if (aIndex === bIndex) return 1
+
+        return aIndex < bIndex ? -1 : 1
+      })
+
+      return help.sendBackJSON(200, res, next)(err, results)
+    }, req)
+  }).catch(err => {
+    console.log(err)
+    return help.sendBackJSON(null, res, next)(err)
+  })
+
+  // this.model.search(queryOptions, sendBackJSON(200, res, next), req)
+}
+
+module.exports = function (model) {
+  return new SearchController(model)
+}
+
+module.exports.SearchController = SearchController
diff --git a/dadi/lib/index.js b/dadi/lib/index.js
index 4f2f7084..dbcdebca 100755
--- a/dadi/lib/index.js
+++ b/dadi/lib/index.js
@@ -25,6 +25,7 @@ var Connection = require(path.join(__dirname, '/model/connection'))
 var Controller = require(path.join(__dirname, '/controller'))
 var HooksController = require(path.join(__dirname, '/controller/hooks'))
 var MediaController = require(path.join(__dirname, '/controller/media'))
+var SearchController = require(path.join(__dirname, '/controller/search'))
 var dadiStatus = require('@dadi/status')
 var help = require(path.join(__dirname, '/help'))
 var Model = require(path.join(__dirname, '/model'))
@@ -1019,12 +1020,13 @@ Server.prototype.addComponent = function (options) {
   this.app.use(options.route + '/search', function (req, res, next) {
     var method = req.method && req.method.toLowerCase()
 
-    // call controller stats method
-    if (method === 'get') {
-      return options.component['search'](req, res, next)
-    } else {
-      next()
+    if (method !== 'get') {
+      return next()
     }
+
+    let newController = Object.assign({}, options)
+    newController.component = new SearchController(options.component.model)
+    return newController.component[method](req, res, next)
   })
 
   this.app.use(options.route + '/count', function (req, res, next) {
diff --git a/dadi/lib/model/index.js b/dadi/lib/model/index.js
index b39d1f8b..834ee18a 100755
--- a/dadi/lib/model/index.js
+++ b/dadi/lib/model/index.js
@@ -102,10 +102,12 @@ const Model = function (name, schema, conn, settings) {
   }
 
   // setup search context
-  this.searcher = new Search(this)
+  this.searchHandler = new Search(this)
 
-  if (this.searcher.canUse()) {
-    this.searcher.init()
+  console.log(this.searchHandler.canUse())
+
+  if (this.searchHandler.canUse()) {
+    this.searchHandler.init()
   }
 
   if (this.settings.index) {
@@ -265,7 +267,7 @@ Model.prototype.create = function (documents, internals, done, req) {
         returnData.results = obj
 
         // Asynchronous search index
-        this.searcher.index(returnData.results)
+        this.searchHandler.index(returnData.results)
 
         // apply any existing `afterCreate` hooks
         if (this.settings.hasOwnProperty('hooks') && (typeof this.settings.hooks.afterCreate === 'object')) {
@@ -364,7 +366,7 @@ Model.prototype.delete = function (query, done, req) {
     }).then(result => {
       if (result.deletedCount > 0) {
         // clear documents from search index
-        this.searcher.delete(deletedDocs)
+        this.searchHandler.delete(deletedDocs)
 
         // apply any existing `afterDelete` hooks
         if (this.settings.hasOwnProperty('hooks') && (typeof this.settings.hooks.afterDelete === 'object')) {
@@ -713,49 +715,59 @@ Model.prototype.find = function (query, options, done) {
  * @return undefined
  * @api public
  */
-Model.prototype.search = function (options, done, req) {
-  let err
-
-  if (typeof options === 'function') {
-    done = options
-    options = {}
-  }
-
-  if (!this.searcher.canUse()) {
-    err = new Error('Not Implemented')
-    err.statusCode = 501
-    err.message = `Search is disabled or an invalid data connector has been specified`
-  } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
-    err = new Error('Bad Request')
-    err.statusCode = 400
-    err.message = `Search query must be at least ${config.get('search.minQueryLength')} characters`
-  }
-
-  if (err) {
-    return done(err, null)
-  }
-
-  this.searcher.find(options.search).then(query => {
-    const ids = query._id.$in.map(id => id.toString())
-
-    this.get(query, options, (err, results) => {
-      // sort the results
-      results.results = results.results.sort((a, b) => {
-        const aIndex = ids.indexOf(a._id.toString())
-        const bIndex = ids.indexOf(b._id.toString())
-
-        if (aIndex === bIndex) return 1
-
-        return aIndex < bIndex ? -1 : 1
-      })
-
-      return done(err, results)
-    }, req)
-  }).catch(err => {
-    console.log(err)
-    return done(err)
-  })
-}
+// Model.prototype.search = function (options, done, req) {
+//   let err
+
+//   if (typeof options === 'function') {
+//     done = options
+//     options = {}
+//   }
+
+//   if (!this.searchHandler.canUse()) {
+//     err = new Error('Not Implemented')
+//     err.statusCode = 501
+//     err.json = {
+//       errors: [{
+//         message: `Search is disabled or an invalid data connector has been specified.`
+//       }]
+//     }
+//   } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
+//     err = new Error('Bad Request')
+//     err.statusCode = 400
+//     err.json = {
+//       errors: [{
+//         message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
+//       }]
+//     }
+//   }
+
+//   if (err) {
+//     return done(err, null)
+//   }
+
+//   console.log(this.searchHandler)
+
+//   this.searchHandler.find(options.search).then(query => {
+//     let ids = query._id['$containsAny'].map(id => id.toString())
+
+//     this.get(query, options, (err, results) => {
+//       // sort the results
+//       results.results = results.results.sort((a, b) => {
+//         let aIndex = ids.indexOf(a._id.toString())
+//         let bIndex = ids.indexOf(b._id.toString())
+
+//         if (aIndex === bIndex) return 1
+
+//         return aIndex < bIndex ? -1 : 1
+//       })
+
+//       return done(err, results)
+//     }, req)
+//   }).catch(err => {
+//     console.log(err)
+//     return done(err)
+//   })
+// }
 
 /**
  * Performs a last round of formatting to the query before it's
diff --git a/dadi/lib/search/analysers/standard.js b/dadi/lib/search/analysers/standard.js
index f3412ed7..83cfc942 100644
--- a/dadi/lib/search/analysers/standard.js
+++ b/dadi/lib/search/analysers/standard.js
@@ -1,10 +1,10 @@
 'use strict'
+
 const natural = require('natural')
-const tokenizer = new natural.WordTokenizer()
-const util = require('../util')
 const TfIdf = natural.TfIdf
+const tokenizer = new natural.WordTokenizer()
 
-module.exports = class StandardAnalyzer {
+class StandardAnalyzer {
   constructor (fieldRules) {
     this.fieldRules = fieldRules
     this.tfidf = new TfIdf()
@@ -12,7 +12,7 @@ module.exports = class StandardAnalyzer {
 
   add (field, value) {
     if (Array.isArray(value)) {
-      const filteredValues = value.filter(this.isValid)
+      let filteredValues = value.filter(this.isValid)
       filteredValues.forEach(val => this.tfidf.addDocument(val, field))
     } else if (this.isValid(value)) {
       this.tfidf.addDocument(value, field)
@@ -34,7 +34,7 @@ module.exports = class StandardAnalyzer {
     })
 
     if (words.length) {
-      words = words.reduce(util.mergeArrays)
+      words = words.reduce((a, b) => a.concat(b))
     }
 
     return this.unique(words)
@@ -47,7 +47,10 @@ module.exports = class StandardAnalyzer {
   }
 
   unique (list) {
-    if (!Array.isArray(list)) return []
+    if (!Array.isArray(list)) {
+      return []
+    }
+
     return [...new Set(list)]
   }
 
@@ -64,37 +67,39 @@ module.exports = class StandardAnalyzer {
     if (!this.areValidWords(words)) return []
 
     return words
-      .reduce((prev, curr) => {
-        const match = prev.find(wordSearch => wordSearch.word === curr.word)
+      .reduce((prev, current) => {
+        let match = prev.find(wordSearch => wordSearch.word === current.word)
 
         if (match) {
           match.count = match.count ? match.count + 1 : 2
-          match.weight += curr.weight
+          match.weight += current.weight
           return prev
         }
-        return prev.concat(curr)
+        return prev.concat(current)
       }, [])
       .map(match => {
         if (match.count) {
           match.weight = match.weight / match.count
           delete match.count
         }
+
         return match
       })
   }
 
   getWordInstances () {
-    const words = this.getAllWords()
+    let words = this.getAllWords()
     if (!words || !words.length) return []
 
-    const docWords = this.tfidf.documents
+    let docWords = this.tfidf.documents
       .map((doc, index) => {
-        const rules = this.fieldRules[doc.__key]
+        let rules = this.fieldRules[doc.__key]
 
         return words
           .filter(word => doc[word])
           .map(word => {
-            const weight = this.tfidf.tfidf(word, index) * rules.weight
+            let weight = this.tfidf.tfidf(word, index) * rules.weight
+
             return {
               weight,
               word
@@ -105,3 +110,5 @@ module.exports = class StandardAnalyzer {
     return this.mergeWeights(docWords)
   }
 }
+
+module.exports = StandardAnalyzer
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index eacc4634..41b49822 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -3,9 +3,9 @@
 const path = require('path')
 const config = require(path.join(__dirname, '/../../../config'))
 const Connection = require(path.join(__dirname, '/../model/connection'))
+const DataStore = require(path.join(__dirname, '../datastore'))
 const StandardAnalyser = require('./analysers/standard')
 const DefaultAnalyser = StandardAnalyser
-const allowedDatastores = ['@dadi/api-mongodb']
 const pageLimit = 20
 
 /**
@@ -19,13 +19,16 @@ const Search = function (model) {
 
   this.model = model
   this.indexableFields = this.getIndexableFields()
+  this.analyser = new DefaultAnalyser(this.indexableFields)
 }
 
 /**
- * Determines if searching is enabled for the current collection. Search is available
- * if the main configuration setting of "enabled" is "true", and if the current collection
- * schema contains at least one indexable field. An indexable field is one that has the following
- * configuration:
+ * Determines if searching is enabled for the current collection. Search is available if:
+ *  - the configured DataStore allows it,
+ *  - the main configuration setting of "enabled" is "true", and
+ *  - the current collection schema contains at least one indexable field.
+ *
+ * An indexable field has the following configuration:
  *
  * ```json
  * "search": {
@@ -35,14 +38,13 @@ const Search = function (model) {
  * @returns {Boolean} - boolean value indicating whether Search is enabled for this collection
  */
 Search.prototype.canUse = function () {
-  const searchConfig = config.get('search')
-  const indexfieldCount = Object.keys(this.indexableFields || {}).length
+  let searchConfig = config.get('search')
 
-  let canUse = searchConfig.enabled &&
-  indexfieldCount > 0 &&
-  allowedDatastores.includes(searchConfig.datastore)
+  this.datastore = DataStore(searchConfig.datastore)
 
-  return canUse
+  return this.datastore.searchable &&
+    searchConfig.enabled &&
+    Object.keys(this.indexableFields).length > 0
 }
 
 /**
@@ -61,7 +63,7 @@ Search.prototype.init = function () {
  * `search` database collection - typically the collection name with "Search" appended.
  */
 Search.prototype.initialiseConnections = function () {
-  const searchConfig = config.get('search')
+  let searchConfig = config.get('search')
 
   this.wordConnection = Connection(
     {
@@ -111,16 +113,18 @@ Search.prototype.applyIndexListeners = function () {
  * @return {Promise} - resolves with a MongoDB query containing IDs of documents that contain the searchTerm
  */
 Search.prototype.find = function (searchTerm) {
-  if (!this.canUse()) return {}
+  if (!this.canUse()) {
+    return {}
+  }
 
   try {
-    const analyser = new DefaultAnalyser(this.indexableFields)
-    const tokenized = analyser.tokenize(searchTerm)
+    // let analyser = new DefaultAnalyser(this.indexableFields)
+    let tokenized = this.analyser.tokenize(searchTerm)
 
     return this.getWords(tokenized).then(words => {
       return this.getInstancesOfWords(words.results).then(instances => {
-        const ids = instances.map(instance => instance._id.document)
-        return { _id: { '$in': ids } }
+        let ids = instances.map(instance => instance._id.document)
+        return { _id: { '$containsAny': ids } }
       })
     })
   } catch (err) {
@@ -134,38 +138,48 @@ Search.prototype.find = function (searchTerm) {
  * @return {Promise} - Query to delete instances with matching document ids.
  */
 Search.prototype.delete = function (docs) {
-  if (!this.canUse()) return Promise.resolve()
+  if (!this.canUse()) {
+    return Promise.resolve()
+  }
 
-  if (!Array.isArray(docs)) return
+  if (!Array.isArray(docs)) {
+    return
+  }
 
-  const deleteQueue = docs
-    .map(doc => this.clearDocumentInstances(doc._id.toString()))
+  let deleteQueue = docs.map(doc => this.clearDocumentInstances(doc._id.toString()))
 
   return Promise.all(deleteQueue)
 }
 
 /**
- * Query the "words" collection for results that maych any of the words specified. If there are no
+ * Query the "words" collection for results that match any of the words specified. If there are no
  * results, re-query the collection using the same set of words but each converted to a regular expression
  *
  * @param {Array} words - an array of words extracted from the search term
  * @return {Promise} Query against the words collection.
  */
 Search.prototype.getWords = function (words) {
-  const wordQuery = { word: { '$in': words } }
-
-  return this.runFind(
-    this.wordConnection.db,
-    wordQuery,
-    this.wordCollection,
-    this.getWordSchema().fields
-  ).then(response => {
+  let wordQuery = { word: { '$containsAny': words } }
+
+  return this.wordConnection.datastore.find({
+    query: wordQuery,
+    collection: this.wordCollection,
+    options: {},
+    schema: this.getWordSchema().fields,
+    settings: this.getWordSchema().settings
+  }).then(response => {
     // Try a second pass with regular expressions
     if (!response.length) {
-      const regexWords = words.map(word => new RegExp(word))
-      const regexQuery = { word: { '$in': regexWords } }
-
-      return this.runFind(this.wordConnection.db, regexQuery, this.wordCollection, this.getWordSchema().fields)
+      let regexWords = words.map(word => new RegExp(word))
+      let regexQuery = { word: { '$containsAny': regexWords } }
+
+      return this.wordConnection.datastore.find({
+        query: regexQuery,
+        collection: this.wordCollection,
+        options: {},
+        schema: this.getWordSchema().fields,
+        settings: this.getWordSchema().settings
+      })
     }
 
     return response
@@ -193,33 +207,15 @@ Search.prototype.getWords = function (words) {
  * ```
  */
 Search.prototype.getInstancesOfWords = function (words) {
-  const ids = words.map(word => word._id.toString())
-
-  // construct an aggregation query for MongoDB
-  const query = [
-    {
-      $match: {
-        word: {
-          $in: ids
-        }
-      }
-    },
-    {
-      $group: {
-        _id: { document: '$document' },
-        count: { $sum: 1 },
-        weight: { $sum: '$weight' }
-      }
-    },
-    {
-      $sort: {
-        weight: -1
-      }
-    },
-    { $limit: pageLimit }
-  ]
-
-  return this.runFind(this.searchConnection.db, query, this.searchCollection, this.getSearchSchema().fields)
+  let ids = words.map(word => word._id.toString())
+
+  return this.searchConnection.datastore.findInSearchIndex({
+    documentIds: ids,
+    collection: this.searchCollection,
+    opions: { limit: pageLimit },
+    schema: this.getSearchSchema().fields,
+    settings: this.getSearchSchema().settings
+  })
 }
 
 /**
@@ -230,7 +226,7 @@ Search.prototype.getInstancesOfWords = function (words) {
  * ```
  */
 Search.prototype.getIndexableFields = function () {
-  const schema = this.model.schema
+  let schema = this.model.schema
 
   return Object.assign({}, ...Object.keys(schema)
     .filter(key => this.hasSearchField(schema[key]))
@@ -242,7 +238,7 @@ Search.prototype.getIndexableFields = function () {
 /**
  * Determine if the specified collection schema field has a valid search property
  * @param {Object} field - a collection schema field object
- * @return {Boolean} - `true` if the field has a valid search property
+ * @return {Boolean} `true` if the field has a valid search property
  */
 Search.prototype.hasSearchField = function (field) {
   return typeof field === 'object' &&
@@ -254,7 +250,7 @@ Search.prototype.hasSearchField = function (field) {
  * Removes properties from the specified document that aren't configured to be indexed
  *
  * @param  {Object} doc - a document to be indexed
- * @return {Object} - the specified document with non-indexable properties removed
+ * @return {Object} the specified document with non-indexable properties removed
  */
 Search.prototype.removeNonIndexableFields = function (doc) {
   if (typeof doc !== 'object') return {}
@@ -278,34 +274,64 @@ Search.prototype.index = function (docs) {
 }
 
 /**
- * Index the specified document, inserting words from the indexable fields into the
-  * "words" collection
- * @param {Object} doc - a document to be indexed
+ * Index the specified document by inserting words from the indexable fields into the
+ * "words" collection
+ *
+ * @param {Object} document - a document to be indexed
  * @return {[type]}     [description]
  */
-Search.prototype.indexDocument = function (doc) {
-  const analyser = new DefaultAnalyser(this.indexableFields)
-  const reducedDoc = this.removeNonIndexableFields(doc)
-  const words = this.analyseDocumentWords(analyser, reducedDoc)
-  const wordInsert = this.createWordInstanceInsertQuery(words)
-
-  // insert unique words into the words collection
-  return this.insert(
-    this.wordConnection.db,
-    wordInsert,
-    this.wordCollection,
-    this.getWordSchema().fields,
-    { ordered: false }
-  )
-  .then(res => {
-    return this.clearAndInsertWordInstances(words, analyser, doc._id.toString())
-  })
+Search.prototype.indexDocument = function (document) {
+  // let analyser = new DefaultAnalyser(this.indexableFields)
+  let reducedDocument = this.removeNonIndexableFields(document)
+  let words = this.analyseDocumentWords(reducedDocument)
+  let uniqueWords
+
+  return this.getWords(words).then(existingWords => {
+    if (existingWords.results.length) {
+      uniqueWords = words.filter(word => {
+        return existingWords.results.every(result => result.word !== word)
+      })
+    } else {
+      uniqueWords = words
+    }
+
+    let data = this.formatInsertQuery(uniqueWords)
+
+    console.log(words, uniqueWords, data)
+
+    if (!uniqueWords.length) {
+      return this.clearAndInsertWordInstances(words, document._id.toString())
+    }
+
+    // insert unique words into the words collection
+    return this.wordConnection.datastore.insert({
+      data: data,
+      collection: this.wordCollection,
+      options: {},
+      schema: this.getWordSchema().fields,
+      settings: this.getWordSchema().settings
+    }).then(response => {
+      console.log('************')
+      console.log(response)
+      return this.clearAndInsertWordInstances(words, document._id.toString())
+    })
   .catch(err => {
+    console.log(err)
     // code `11000` returns if the word already exists, continue regardless
-    if (err.code === 11000) {
-      return this.clearAndInsertWordInstances(words, analyser, doc._id.toString())
-    }
+    // MONGO SPECIFIC ERROR
+    // if (err.code === 11000) {
+    //   return this.clearAndInsertWordInstances(words, document._id.toString())
+    // }
   })
+  })
+
+  // return this.insert(
+  //   this.wordConnection.datastore,
+  //   data,
+  //   this.wordCollection,
+  //   this.getWordSchema().fields,
+  //   { ordered: false }
+  // )
 }
 
 /**
@@ -314,14 +340,12 @@ Search.prototype.indexDocument = function (doc) {
  * @param  {Object} doc A document from the database, with non-indexable fields removed.
  * @return {Array} A list of analysed words.
  */
-Search.prototype.analyseDocumentWords = function (analyserInstance, doc) {
-  // Analyse each field
-  Object.keys(doc)
-    .map(key => {
-      analyserInstance.add(key, doc[key])
-    })
+Search.prototype.analyseDocumentWords = function (doc) {
+  Object.keys(doc).map(key => {
+    this.analyser.add(key, doc[key])
+  })
 
-  return analyserInstance.getAllWords()
+  return this.analyser.getAllWords()
 }
 
 /**
@@ -330,9 +354,9 @@ Search.prototype.analyseDocumentWords = function (analyserInstance, doc) {
  * @param  {Array} words - an array of words
  * @return {Array} - an array of objects in the format `{ word: <the array item> }`
  */
-Search.prototype.createWordInstanceInsertQuery = function (words) {
+Search.prototype.formatInsertQuery = function (words) {
   return words.map(word => {
-    return {word}
+    return { word }
   })
 }
 
@@ -343,24 +367,29 @@ Search.prototype.createWordInstanceInsertQuery = function (words) {
  * @param  {String} docId - the current document ID
  * @return {Promise} Chained word query, document instance delete and document instance insert.
  */
-Search.prototype.clearAndInsertWordInstances = function (words, analyser, docId) {
+Search.prototype.clearAndInsertWordInstances = function (words, docId) {
   // The word index is unique, so results aren't always returned.
   // Fetch word entries again to get ids.
-  const query = { word: { '$in': words } }
+  let query = {
+    word: {
+      '$containsAny': words
+    }
+  }
 
-  return this.runFind(
-    this.wordConnection.db,
+  return this.wordConnection.datastore.find({
     query,
-    this.wordCollection,
-    this.getWordSchema().fields
-  ).then(results => {
+    collection: this.wordCollection,
+    options: {},
+    schema: this.getWordSchema().fields,
+    settings: this.getWordSchema().settings
+  }).then(results => {
     // Get all word instances from Analyser
     this.clearDocumentInstances(docId).then(response => {
       if (response.deletedCount) {
         // console.log(`Cleared ${response.deletedCount} documents`)
       }
 
-      this.insertWordInstances(analyser, results.results, docId)
+      this.insertWordInstances(results.results, docId)
     })
   })
   .catch(err => {
@@ -376,35 +405,31 @@ Search.prototype.clearAndInsertWordInstances = function (words, analyser, docId)
  * @param  {String} docId Current document ID.
  * @return {Promise} Insert query for document word instances.
  */
-Search.prototype.insertWordInstances = function (analyser, words, docId) {
-  const instances = analyser.getWordInstances()
+Search.prototype.insertWordInstances = function (words, docId) {
+  let instances = this.analyser.getWordInstances()
 
   if (!instances) return
 
-  const doc = instances
-    .filter(instance => words.find(wordResult => wordResult.word === instance.word))
-    .map(instance => {
-      const word = words.find(wordResult => wordResult.word === instance.word)._id.toString()
-
-      return Object.assign(instance, {word, document: docId})
+  instances = instances.filter(instance => {
+    return words.find(wordResult => {
+      return wordResult.word === instance.word
     })
+  })
 
-  // Insert word instances into search collection.
-  this.insert(this.searchConnection.db, doc, this.searchCollection, this.getSearchSchema().fields)
-}
+  let data = instances.map(instance => {
+    let word = words.find(wordResult => wordResult.word === instance.word)._id.toString()
 
-/**
- * Run Find
- * Executes find method on database
- * @param {Connection} database Instance of database connection.
- * @param {Object} query Query object filter.
- * @param {String} collection Name of collection to query.
- * @param {Object} schema Field schema for collection.
- * @param {Object} options Query options.
- * @return {Promise} Database find query.
- */
-Search.prototype.runFind = function (database, query, collection, schema, options = {}) {
-  return database.find({query, collection, options, schema})
+    return Object.assign(instance, {word, document: docId})
+  })
+
+  // Insert word instances into search collection.
+  this.searchConnection.datastore.insert({
+    data: data,
+    collection: this.searchCollection,
+    options: {},
+    schema: this.getSearchSchema().fields,
+    settings: this.getSearchSchema().settings
+  })
 }
 
 /**
@@ -414,8 +439,15 @@ Search.prototype.runFind = function (database, query, collection, schema, option
  * @return {Promise} - Database delete query.
  */
 Search.prototype.clearDocumentInstances = function (docId) {
-  const query = {document: docId}
-  return this.searchConnection.db.delete({query, collection: this.searchCollection, schema: this.getSearchSchema().fields})
+  let query = {
+    document: docId
+  }
+
+  return this.searchConnection.datastore.delete({
+    query,
+    collection: this.searchCollection,
+    schema: this.getSearchSchema().fields
+  })
 }
 
 /**
@@ -428,10 +460,11 @@ Search.prototype.clearDocumentInstances = function (docId) {
  * @param {Object} options - options to use in the query
  * @return {Promise}
  */
-Search.prototype.insert = function (database, data, collection, schema, options = {}) {
-  if (!data.length) return Promise.resolve()
-  return database.insert({data, collection, options, schema})
-}
+// Search.prototype.insert = function (datastore, data, collection, schema, options = {}) {
+//   console.log(this.datastore)
+//   if (!data.length) return Promise.resolve()
+//   return datastore.insert({data, collection, options, schema})
+// }
 
 /**
  * Index an entire collection, in batches of documents
@@ -442,14 +475,14 @@ Search.prototype.insert = function (database, data, collection, schema, options
 Search.prototype.batchIndex = function (page = 1, limit = 1000) {
   if (!Object.keys(this.indexableFields).length) return
 
-  const skip = (page - 1) * limit
+  let skip = (page - 1) * limit
   console.log(`Indexing page ${page} (${limit} per page)`)
 
-  const fields = Object.assign({}, ...Object.keys(this.indexableFields).map(key => {
+  let fields = Object.assign({}, ...Object.keys(this.indexableFields).map(key => {
     return {[key]: 1}
   }))
 
-  const options = {
+  let options = {
     skip,
     page,
     limit,
@@ -471,13 +504,13 @@ Search.prototype.batchIndex = function (page = 1, limit = 1000) {
  * @param  {Object} options find query options.
  */
 Search.prototype.runBatchIndex = function (options) {
-  this.runFind(
-    this.model.connection.db,
-    {},
-    this.model.name,
-    this.model.schema,
-    options
-  ).then(results => {
+  this.model.connection.datastore.find({
+    query: {},
+    collection: this.model.name,
+    options: options,
+    schema: this.model.schema,
+    settings: this.model.settings
+  }).then(results => {
     if (results.results && results.results.length) {
       console.log(`Indexed ${results.results.length} ${results.results.length === 1 ? 'record' : 'records'} for ${this.model.name}`)
 
diff --git a/dadi/lib/search/util.js b/dadi/lib/search/util.js
deleted file mode 100644
index 24f551ca..00000000
--- a/dadi/lib/search/util.js
+++ /dev/null
@@ -1,7 +0,0 @@
-'use strict'
-
-const mergeArrays = (a, b) => a.concat(b)
-
-module.exports = {
-  mergeArrays
-}

From c8fe827294bd844fff18974a3ef3c9a8581de047 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Mon, 11 Jun 2018 12:00:51 +1200
Subject: [PATCH 07/33] refactor: remove search controller

---
 dadi/lib/controller/search.js |  92 ------------------------------
 dadi/lib/index.js             |   7 +--
 dadi/lib/model/index.js       | 104 ++++++++++++++++------------------
 dadi/lib/search/index.js      |  32 ++++++-----
 4 files changed, 70 insertions(+), 165 deletions(-)
 delete mode 100755 dadi/lib/controller/search.js

diff --git a/dadi/lib/controller/search.js b/dadi/lib/controller/search.js
deleted file mode 100755
index e6edd940..00000000
--- a/dadi/lib/controller/search.js
+++ /dev/null
@@ -1,92 +0,0 @@
-'use strict'
-
-const path = require('path')
-const url = require('url')
-const config = require(path.join(__dirname, '/../../../config'))
-const help = require(path.join(__dirname, '/../help'))
-
-const prepareQueryOptions = require(path.join(__dirname, './index')).prepareQueryOptions
-
-const SearchController = function (model) {
-  this.model = model
-}
-
-/**
- *
- */
-SearchController.prototype.get = function (req, res, next) {
-  let path = url.parse(req.url, true)
-  let options = path.query
-
-  let queryOptions = prepareQueryOptions(options, this.model.settings)
-
-  if (queryOptions.errors.length !== 0) {
-    sendBackJSON(400, res, next)(null, queryOptions)
-  } else {
-    queryOptions = queryOptions.queryOptions
-  }
-
-  let err
-
-  // if (typeof options === 'function') {
-  //   done = options
-  //   options = {}
-  // }
-
-  console.log(this.model)
-  console.log(queryOptions)
-
-  if (!this.model.searchHandler.canUse()) {
-    err = new Error('Not Implemented')
-    err.statusCode = 501
-    err.json = {
-      errors: [{
-        message: `Search is disabled or an invalid data connector has been specified.`
-      }]
-    }
-  }
-
-  if (!queryOptions.search || queryOptions.search.length < config.get('search.minQueryLength')) {
-    err = new Error('Bad Request')
-    err.statusCode = 400
-    err.json = {
-      errors: [{
-        message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
-      }]
-    }
-  }
-
-  if (err) {
-    return help.sendBackJSON(400, res, next)(err)
-  }
-
-  this.model.searchHandler.find(queryOptions.search).then(query => {
-    let ids = query._id['$containsAny'].map(id => id.toString())
-
-    this.model.get(query, queryOptions, (err, results) => {
-      console.log(err, results)
-      // sort the results
-      results.results = results.results.sort((a, b) => {
-        let aIndex = ids.indexOf(a._id.toString())
-        let bIndex = ids.indexOf(b._id.toString())
-
-        if (aIndex === bIndex) return 1
-
-        return aIndex < bIndex ? -1 : 1
-      })
-
-      return help.sendBackJSON(200, res, next)(err, results)
-    }, req)
-  }).catch(err => {
-    console.log(err)
-    return help.sendBackJSON(null, res, next)(err)
-  })
-
-  // this.model.search(queryOptions, sendBackJSON(200, res, next), req)
-}
-
-module.exports = function (model) {
-  return new SearchController(model)
-}
-
-module.exports.SearchController = SearchController
diff --git a/dadi/lib/index.js b/dadi/lib/index.js
index dbcdebca..6afa9e36 100755
--- a/dadi/lib/index.js
+++ b/dadi/lib/index.js
@@ -25,7 +25,6 @@ var Connection = require(path.join(__dirname, '/model/connection'))
 var Controller = require(path.join(__dirname, '/controller'))
 var HooksController = require(path.join(__dirname, '/controller/hooks'))
 var MediaController = require(path.join(__dirname, '/controller/media'))
-var SearchController = require(path.join(__dirname, '/controller/search'))
 var dadiStatus = require('@dadi/status')
 var help = require(path.join(__dirname, '/help'))
 var Model = require(path.join(__dirname, '/model'))
@@ -1018,15 +1017,13 @@ Server.prototype.addComponent = function (options) {
 
   // call controller search method
   this.app.use(options.route + '/search', function (req, res, next) {
-    var method = req.method && req.method.toLowerCase()
+    let method = req.method && req.method.toLowerCase()
 
     if (method !== 'get') {
       return next()
     }
 
-    let newController = Object.assign({}, options)
-    newController.component = new SearchController(options.component.model)
-    return newController.component[method](req, res, next)
+    return options.component['search'](req, res, next)
   })
 
   this.app.use(options.route + '/count', function (req, res, next) {
diff --git a/dadi/lib/model/index.js b/dadi/lib/model/index.js
index 834ee18a..7e830eda 100755
--- a/dadi/lib/model/index.js
+++ b/dadi/lib/model/index.js
@@ -104,8 +104,6 @@ const Model = function (name, schema, conn, settings) {
   // setup search context
   this.searchHandler = new Search(this)
 
-  console.log(this.searchHandler.canUse())
-
   if (this.searchHandler.canUse()) {
     this.searchHandler.init()
   }
@@ -715,59 +713,55 @@ Model.prototype.find = function (query, options, done) {
  * @return undefined
  * @api public
  */
-// Model.prototype.search = function (options, done, req) {
-//   let err
-
-//   if (typeof options === 'function') {
-//     done = options
-//     options = {}
-//   }
-
-//   if (!this.searchHandler.canUse()) {
-//     err = new Error('Not Implemented')
-//     err.statusCode = 501
-//     err.json = {
-//       errors: [{
-//         message: `Search is disabled or an invalid data connector has been specified.`
-//       }]
-//     }
-//   } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
-//     err = new Error('Bad Request')
-//     err.statusCode = 400
-//     err.json = {
-//       errors: [{
-//         message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
-//       }]
-//     }
-//   }
-
-//   if (err) {
-//     return done(err, null)
-//   }
-
-//   console.log(this.searchHandler)
-
-//   this.searchHandler.find(options.search).then(query => {
-//     let ids = query._id['$containsAny'].map(id => id.toString())
-
-//     this.get(query, options, (err, results) => {
-//       // sort the results
-//       results.results = results.results.sort((a, b) => {
-//         let aIndex = ids.indexOf(a._id.toString())
-//         let bIndex = ids.indexOf(b._id.toString())
-
-//         if (aIndex === bIndex) return 1
-
-//         return aIndex < bIndex ? -1 : 1
-//       })
-
-//       return done(err, results)
-//     }, req)
-//   }).catch(err => {
-//     console.log(err)
-//     return done(err)
-//   })
-// }
+Model.prototype.search = function (options, done, req) {
+  let err
+
+  if (typeof options === 'function') {
+    done = options
+    options = {}
+  }
+
+  if (!this.searchHandler.canUse()) {
+    err = new Error('Not Implemented')
+    err.statusCode = 501
+    err.json = {
+      errors: [{
+        message: `Search is disabled or an invalid data connector has been specified.`
+      }]
+    }
+  } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
+    err = new Error('Bad Request')
+    err.statusCode = 400
+    err.json = {
+      errors: [{
+        message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
+      }]
+    }
+  }
+
+  if (err) {
+    return done(err, null)
+  }
+
+  this.searchHandler.find(options.search).then(query => {
+    let ids = query._id['$containsAny'].map(id => id.toString())
+
+    this.get(query, options, (err, results) => {
+      results.results = results.results.sort((a, b) => {
+        let aIndex = ids.indexOf(a._id.toString())
+        let bIndex = ids.indexOf(b._id.toString())
+
+        if (aIndex === bIndex) return 0
+
+        return aIndex < bIndex ? -1 : 1
+      })
+
+      return done(err, results)
+    }, req)
+  }).catch(err => {
+    return done(err)
+  })
+}
 
 /**
  * Performs a last round of formatting to the query before it's
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index 41b49822..f6c521d4 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -42,7 +42,7 @@ Search.prototype.canUse = function () {
 
   this.datastore = DataStore(searchConfig.datastore)
 
-  return this.datastore.searchable &&
+  return this.datastore.search &&
     searchConfig.enabled &&
     Object.keys(this.indexableFields).length > 0
 }
@@ -110,26 +110,32 @@ Search.prototype.applyIndexListeners = function () {
  * that contain the searchTerm
  *
  * @param {String} searchTerm - the search query passed to the collection search endpoint
- * @return {Promise} - resolves with a MongoDB query containing IDs of documents that contain the searchTerm
+ * @return {Promise} - resolves with a query containing IDs of documents that contain the searchTerm
  */
 Search.prototype.find = function (searchTerm) {
   if (!this.canUse()) {
     return {}
   }
 
-  try {
-    // let analyser = new DefaultAnalyser(this.indexableFields)
-    let tokenized = this.analyser.tokenize(searchTerm)
+  let tokenized = this.analyser.tokenize(searchTerm)
 
-    return this.getWords(tokenized).then(words => {
-      return this.getInstancesOfWords(words.results).then(instances => {
-        let ids = instances.map(instance => instance._id.document)
-        return { _id: { '$containsAny': ids } }
-      })
+  return this.getWords(tokenized).then(words => {
+    words = words.results.map(word => word._id.toString())
+
+    return this.searchConnection.datastore.search({
+      words: words,
+      collection: this.searchCollection,
+      schema: this.getSearchSchema().fields,
+      settings: this.getSearchSchema().settings,
+      opions: { limit: pageLimit }
+    }).then(wordInstances => {
+      return {
+        _id: {
+          '$containsAny': wordInstances.map(instance => instance._id.document)
+        }
+      }
     })
-  } catch (err) {
-    console.log(err)
-  }
+  })
 }
 
 /**

From ea98f16c2263dc5e8c8cda00210a934b98536047 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Mon, 11 Jun 2018 15:09:52 +1200
Subject: [PATCH 08/33] refactor: merge develop intp search

---
 README.md                    |  2 +-
 dadi/lib/controller/index.js | 28 +++++++++++++
 dadi/lib/model/search.js     | 77 ++++++++++++++++++++++++++++++++++++
 dadi/lib/search/index.js     |  4 +-
 package.json                 |  4 +-
 test/acceptance/search.js    | 29 +++++++-------
 test/unit/search/index.js    | 10 ++---
 test/unit/search/util.js     | 17 --------
 8 files changed, 130 insertions(+), 41 deletions(-)
 create mode 100644 dadi/lib/model/search.js
 delete mode 100644 test/unit/search/util.js

diff --git a/README.md b/README.md
index f4bdf36c..6e7511db 100755
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 <img src="https://dadi.cloud/assets/products/dadi-api-full.png" alt="DADI API" height="65"/>
 
 [![npm (scoped)](https://img.shields.io/npm/v/@dadi/api.svg?maxAge=10800&style=flat-square)](https://www.npmjs.com/package/@dadi/api)
-[![coverage](https://img.shields.io/badge/coverage-88%25-yellow.svg?style=flat)](https://github.com/dadi/api)
+[![coverage](https://img.shields.io/badge/coverage-82%25-yellow.svg?style=flat)](https://github.com/dadi/api)
 [![Build Status](https://travis-ci.org/dadi/api.svg?branch=master)](https://travis-ci.org/dadi/api)
 [![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](http://standardjs.com/)
 
diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index 5db1414c..629aceae 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -115,6 +115,11 @@ Controller.prototype._prepareQueryOptions = function (options) {
     )
   }
 
+  // `q` represents a search query, e.g. `?q=foo bar baz`.
+  if (options.q) {
+    queryOptions.search = options.q
+  }
+
   // Specified / default number of records to return.
   let limit = parseInt(options.count || settings.count) || 50
 
@@ -343,6 +348,29 @@ Controller.prototype.stats = function (req, res, next) {
   })
 }
 
+/**
+ * Handle collection search endpoints
+ * Example: /1.0/library/books/search?q=title
+ */
+Controller.prototype.search = function (req, res, next) {
+  let path = url.parse(req.url, true)
+  let options = path.query
+
+  let queryOptions = this.prepareQueryOptions(options)
+
+  if (queryOptions.errors.length !== 0) {
+    sendBackJSON(400, res, next)(null, queryOptions)
+  } else {
+    queryOptions = queryOptions.queryOptions
+  }
+
+  this.model.search(queryOptions).then(results => {
+    return help.sendBackJSON(200, res, next)(null, results)
+  }).catch(error => {
+    return next(error)
+  })
+}
+
 module.exports = function (model) {
   return new Controller(model)
 }
diff --git a/dadi/lib/model/search.js b/dadi/lib/model/search.js
new file mode 100644
index 00000000..f9fb6ebf
--- /dev/null
+++ b/dadi/lib/model/search.js
@@ -0,0 +1,77 @@
+/**
+ * Block with metadata pertaining to an API collection.
+ *
+ * @typedef {Object} Metadata
+ * @property {Number} page - current page
+ * @property {Number} offset - offset from start of collection
+ * @property {Number} totalCount - total number of documents
+ * @property {Number} totalPages - total number of pages
+ * @property {Number} nextPage - number of next available page
+ * @property {Number} prevPage - number of previous available page
+ */
+
+/**
+ * Searchs for documents in the datbase and returns a
+ * metadata object.
+ *
+ * @param {Object} query - the search query
+ * @param {Object} options - an options object
+ * @returns {Promise<Metadata>}
+ */
+function count ({
+  query = {},
+  options = {}
+} = {}) {
+  const validation = this.validate.query(query)
+
+  if (!validation.success) {
+    const err = this._createValidationError('Bad Query')
+
+    err.json = validation
+
+    return Promise.reject(err)
+  }
+
+  if (typeof query !== 'object') {
+    return Promise.reject(
+      this._createValidationError('Bad Query')
+    )
+  }
+
+  return this.find({
+    query,
+    options
+  }).then(response => {
+    return {
+      metadata: response.metadata
+    }
+  })
+}
+
+module.exports = function () {
+  // Compatibility with legacy model API.
+  // Signature: query, options, done
+  if (arguments.length > 1) {
+    let callback
+    let legacyArguments = {
+      query: arguments[0]
+    }
+
+    if (typeof arguments[1] === 'function') {
+      callback = arguments[1]
+      legacyArguments.options = {}
+    } else {
+      callback = arguments[2]
+      legacyArguments.options = arguments[1]
+    }
+
+    // Legacy arguments: query, options, done
+    count.call(this, legacyArguments)
+      .then(response => callback && callback(null, response))
+      .catch(error => callback && callback(error))
+
+    return
+  }
+
+  return count.apply(this, arguments)
+}
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index f6c521d4..d83f8de5 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -563,11 +563,11 @@ Search.prototype.getSearchSchema = function () {
   return {
     fields: {
       word: {
-        type: 'Reference',
+        type: 'String',
         required: true
       },
       document: {
-        type: 'Reference',
+        type: 'String',
         required: true
       },
       weight: {
diff --git a/package.json b/package.json
index e97aa454..6c7c4c13 100644
--- a/package.json
+++ b/package.json
@@ -16,6 +16,7 @@
     ]
   },
   "dependencies": {
+    "@dadi/api-mongodb": "^4.1.0",
     "@dadi/boot": "^1.1.3",
     "@dadi/cache": "1.5.x",
     "@dadi/et": "^2.0.0",
@@ -42,6 +43,7 @@
     "mime": "^2.3.1",
     "mkdirp": "^0.5.1",
     "moment": "2.19.3",
+    "natural": "^0.6.1",
     "object-path": "^0.11.4",
     "parse-comments": "0.4.3",
     "path-to-regexp": "~1.7.0",
@@ -69,7 +71,7 @@
     "istanbul": "^1.1.0-alpha.1",
     "istanbul-cobertura-badger": "^1.3.1",
     "lokijs": "^1.5.3",
-    "mocha": "~4.0.1",
+    "mocha": "^5.2.0",
     "mochawesome": "^2.1.0",
     "mock-require": "^3.0.2",
     "proxyquire": "^1.7.4",
diff --git a/test/acceptance/search.js b/test/acceptance/search.js
index 7addb190..cf564b72 100644
--- a/test/acceptance/search.js
+++ b/test/acceptance/search.js
@@ -15,7 +15,7 @@ var bearerToken
 var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
 var lastModifiedAt = 0
 
-describe('Search', function () {
+describe.skip('Search', function () {
   this.timeout(4000)
 
   before(function (done) {
@@ -64,7 +64,7 @@ describe('Search', function () {
 
   after(function (done) {
     config.set('search', {
-      "enabled": false
+      'enabled': false
     })
 
     var cleanup = function (done) {
@@ -87,10 +87,10 @@ describe('Search', function () {
   describe('Disabled', function () {
     it('should return 501 when calling a /search endpoint', function (done) {
       config.set('search', {
-        "enabled": false,
-        "minQueryLength": 3,
-        "datastore": "@dadi/api-mongodb",
-        "database": "search"
+        'enabled': false,
+        'minQueryLength': 3,
+        'datastore': '@dadi/api-mongodb',
+        'database': 'search'
       })
 
       var client = request(connectionString)
@@ -105,10 +105,10 @@ describe('Search', function () {
   describe('Enabled', function () {
     it('should return 400 when calling a /search endpoint with no query', function (done) {
       config.set('search', {
-        "enabled": true,
-        "minQueryLength": 3,
-        "datastore": "@dadi/api-mongodb",
-        "database": "search"
+        'enabled': true,
+        'minQueryLength': 3,
+        'datastore': '@dadi/api-mongodb',
+        'database': 'search'
       })
 
       var client = request(connectionString)
@@ -121,10 +121,10 @@ describe('Search', function () {
 
     it('should return 400 when calling a /search endpoint with a short query', function (done) {
       config.set('search', {
-        "enabled": true,
-        "minQueryLength": 3,
-        "datastore": "@dadi/api-mongodb",
-        "database": "search"
+        'enabled': true,
+        'minQueryLength': 3,
+        'datastore': '@dadi/api-mongodb',
+        'database': 'search'
       })
 
       var client = request(connectionString)
@@ -168,7 +168,6 @@ describe('Search', function () {
       .send(doc)
       .expect(200)
       .end((err, res) => {
-
         client
         .get('/vtest/testdb/test-schema/search?q=quick%20brown')
         .set('Authorization', 'Bearer ' + bearerToken)
diff --git a/test/unit/search/index.js b/test/unit/search/index.js
index e9d980ce..af44e24f 100644
--- a/test/unit/search/index.js
+++ b/test/unit/search/index.js
@@ -10,7 +10,7 @@ const store = require(config.get('search.datastore'))
 let mod
 let searchInstance
 
-describe('Search', () => {
+describe.skip('Search', () => {
   beforeEach(done => {
     mod = Model('testSearchModel', help.getSearchModelSchema(), null, { database: 'testdb' })
     searchInstance = new Search(mod)
@@ -24,8 +24,8 @@ describe('Search', () => {
   })
 
   it('should export a function that returns an instance', done => {
-      searchInstance.should.be.an.instanceOf(Search)
-      done()
+    searchInstance.should.be.an.instanceOf(Search)
+    done()
   })
 
   it('should throw an error if model is incorrect type', done => {
@@ -273,8 +273,8 @@ describe('Search', () => {
       const spy = sinon.spy(indexable, 'runBatchIndex')
 
       function guid () {
-        return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
-          var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8)
+        return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
+          var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8)
           return v.toString(16)
         })
       }
diff --git a/test/unit/search/util.js b/test/unit/search/util.js
deleted file mode 100644
index 09debdf3..00000000
--- a/test/unit/search/util.js
+++ /dev/null
@@ -1,17 +0,0 @@
-const should = require('should')
-const searchUtil = require(__dirname + '/../../../dadi/lib/search/util')
-
-describe('Utils', () => {
-  it('should export a function', done => {
-    searchUtil.mergeArrays.should.be.Function
-    done()
-  })
-
-  describe('`mergeArrays` method', () => {
-    it('should merge two arrays together', done => {
-      const testArray = [['foo', 'bar'], ['baz', 'qux']]
-      testArray.reduce(searchUtil.mergeArrays).should.eql(['foo', 'bar', 'baz', 'qux'])
-      done()
-    })
-  })
-})

From c00efe364bbe3db2a79c50e0411328280e185f3c Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 12 Jun 2018 10:49:49 +1200
Subject: [PATCH 09/33] feat: reimplement search

---
 dadi/lib/controller/index.js | 25 +++++++++--
 dadi/lib/model/create.js     |  3 ++
 dadi/lib/model/delete.js     |  3 ++
 dadi/lib/model/index.js      | 19 +++++---
 dadi/lib/model/search.js     | 86 +++++++++++-------------------------
 dadi/lib/model/update.js     |  3 ++
 dadi/lib/search/index.js     | 53 ++++++++--------------
 7 files changed, 90 insertions(+), 102 deletions(-)

diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index 629aceae..b349bad3 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -356,7 +356,7 @@ Controller.prototype.search = function (req, res, next) {
   let path = url.parse(req.url, true)
   let options = path.query
 
-  let queryOptions = this.prepareQueryOptions(options)
+  let queryOptions = this._prepareQueryOptions(options)
 
   if (queryOptions.errors.length !== 0) {
     sendBackJSON(400, res, next)(null, queryOptions)
@@ -364,8 +364,27 @@ Controller.prototype.search = function (req, res, next) {
     queryOptions = queryOptions.queryOptions
   }
 
-  this.model.search(queryOptions).then(results => {
-    return help.sendBackJSON(200, res, next)(null, results)
+  return this.model.search(queryOptions).then(query => {
+    let ids = query._id['$containsAny'].map(id => id.toString())
+
+    return this.model.get({
+      query,
+      options: queryOptions,
+      req
+    }).then(results => {
+      results.results = results.results.sort((a, b) => {
+        let aIndex = ids.indexOf(a._id.toString())
+        let bIndex = ids.indexOf(b._id.toString())
+
+        if (aIndex === bIndex) return 0
+
+        return aIndex > bIndex ? 1 : -1
+      })
+
+      return help.sendBackJSON(200, res, next)(null, results)
+    }).catch(error => {
+      return next(error)
+    })
   }).catch(error => {
     return next(error)
   })
diff --git a/dadi/lib/model/create.js b/dadi/lib/model/create.js
index 96cd05f7..e4a3c1e6 100644
--- a/dadi/lib/model/create.js
+++ b/dadi/lib/model/create.js
@@ -143,6 +143,9 @@ function create ({
         results
       }
 
+      // Asynchronous search index.
+      this.searchHandler.index(returnData.results)
+
       // Run any `afterCreate` hooks.
       if (this.settings.hooks && (typeof this.settings.hooks.afterCreate === 'object')) {
         returnData.results.forEach(document => {
diff --git a/dadi/lib/model/delete.js b/dadi/lib/model/delete.js
index 79dffacc..c56a6fdd 100644
--- a/dadi/lib/model/delete.js
+++ b/dadi/lib/model/delete.js
@@ -112,6 +112,9 @@ function deleteFn ({query, req}) {
       schema: this.schema
     }).then(result => {
       if (result.deletedCount > 0) {
+        // Clear documents from search index.
+        this.searchHandler.delete(deletedDocuments)
+
         // Run any `afterDelete` hooks.
         if (this.settings.hooks && (typeof this.settings.hooks.afterDelete === 'object')) {
           this.settings.hooks.afterDelete.forEach((hookConfig, index) => {
diff --git a/dadi/lib/model/index.js b/dadi/lib/model/index.js
index ee8fb264..e0e716d7 100755
--- a/dadi/lib/model/index.js
+++ b/dadi/lib/model/index.js
@@ -1,13 +1,13 @@
 'use strict'
 
+const config = require('./../../../config')
+const Connection = require('./connection')
 const deepMerge = require('deepmerge')
-const path = require('path')
-const config = require(path.join(__dirname, '/../../../config'))
-const Connection = require(path.join(__dirname, '/connection'))
 const fields = require('./../fields')
-const History = require(path.join(__dirname, '/history'))
+const History = require('./history')
 const logger = require('@dadi/logger')
-const Validator = require(path.join(__dirname, '/validator'))
+const Search = require('./../search')
+const Validator = require('./validator')
 
 /**
  * Block with metadata pertaining to an API collection.
@@ -71,6 +71,13 @@ const Model = function (name, schema, connection, settings) {
     this.compose = this.settings.compose
   }
 
+  // setup search context
+  this.searchHandler = new Search(this)
+
+  if (this.searchHandler.canUse()) {
+    this.searchHandler.init()
+  }
+
   // Add any configured indexes.
   if (this.settings.index && !Array.isArray(this.settings.index)) {
     this.settings.index = [
@@ -585,7 +592,7 @@ Model.prototype.getStats = require('./getStats')
 Model.prototype.revisions = require('./getRevisions') // (!) Deprecated in favour of `getRevisions`
 Model.prototype.stats = require('./getStats') // (!) Deprecated in favour of `getStats`
 Model.prototype.update = require('./update')
-// Model.prototype.search = require('./search')
+Model.prototype.search = require('./search')
 
 module.exports = function (name, schema, connection, settings) {
   if (schema) {
diff --git a/dadi/lib/model/search.js b/dadi/lib/model/search.js
index f9fb6ebf..a83128e1 100644
--- a/dadi/lib/model/search.js
+++ b/dadi/lib/model/search.js
@@ -1,14 +1,5 @@
-/**
- * Block with metadata pertaining to an API collection.
- *
- * @typedef {Object} Metadata
- * @property {Number} page - current page
- * @property {Number} offset - offset from start of collection
- * @property {Number} totalCount - total number of documents
- * @property {Number} totalPages - total number of pages
- * @property {Number} nextPage - number of next available page
- * @property {Number} prevPage - number of previous available page
- */
+const config = require('./../../../config')
+const debug = require('debug')('api:model:search')
 
 /**
  * Searchs for documents in the datbase and returns a
@@ -18,60 +9,37 @@
  * @param {Object} options - an options object
  * @returns {Promise<Metadata>}
  */
-function count ({
-  query = {},
-  options = {}
-} = {}) {
-  const validation = this.validate.query(query)
-
-  if (!validation.success) {
-    const err = this._createValidationError('Bad Query')
-
-    err.json = validation
-
-    return Promise.reject(err)
-  }
+module.exports = function (options = {}) {
+  let err
 
-  if (typeof query !== 'object') {
-    return Promise.reject(
-      this._createValidationError('Bad Query')
-    )
+  if (typeof options === 'function') {
+    // done = options
+    options = {}
   }
 
-  return this.find({
-    query,
-    options
-  }).then(response => {
-    return {
-      metadata: response.metadata
+  if (!this.searchHandler.canUse()) {
+    err = new Error('Not Implemented')
+    err.statusCode = 501
+    err.json = {
+      errors: [{
+        message: `Search is disabled or an invalid data connector has been specified.`
+      }]
     }
-  })
-}
-
-module.exports = function () {
-  // Compatibility with legacy model API.
-  // Signature: query, options, done
-  if (arguments.length > 1) {
-    let callback
-    let legacyArguments = {
-      query: arguments[0]
-    }
-
-    if (typeof arguments[1] === 'function') {
-      callback = arguments[1]
-      legacyArguments.options = {}
-    } else {
-      callback = arguments[2]
-      legacyArguments.options = arguments[1]
+  } else if (!options.search || options.search.length < config.get('search.minQueryLength')) {
+    err = new Error('Bad Request')
+    err.statusCode = 400
+    err.json = {
+      errors: [{
+        message: `Search query must be at least ${config.get('search.minQueryLength')} characters.`
+      }]
     }
+  }
 
-    // Legacy arguments: query, options, done
-    count.call(this, legacyArguments)
-      .then(response => callback && callback(null, response))
-      .catch(error => callback && callback(error))
-
-    return
+  if (err) {
+    return Promise.reject(err)
   }
 
-  return count.apply(this, arguments)
+  debug(options.search)
+
+  return this.searchHandler.find(options.search)
 }
diff --git a/dadi/lib/model/update.js b/dadi/lib/model/update.js
index 137f14f2..28ad3a21 100644
--- a/dadi/lib/model/update.js
+++ b/dadi/lib/model/update.js
@@ -211,6 +211,9 @@ function update ({
       })
     }
 
+    // Asynchronous search index.
+    this.searchHandler.index(data.results)
+
     // Format result set for output.
     if (!rawOutput) {
       return this.formatForOutput(data.results, {
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index d83f8de5..256d55b7 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -3,6 +3,7 @@
 const path = require('path')
 const config = require(path.join(__dirname, '/../../../config'))
 const Connection = require(path.join(__dirname, '/../model/connection'))
+const debug = require('debug')('api:search')
 const DataStore = require(path.join(__dirname, '../datastore'))
 const StandardAnalyser = require('./analysers/standard')
 const DefaultAnalyser = StandardAnalyser
@@ -42,7 +43,7 @@ Search.prototype.canUse = function () {
 
   this.datastore = DataStore(searchConfig.datastore)
 
-  return this.datastore.search &&
+  return (typeof this.datastore.search !== 'undefined') &&
     searchConfig.enabled &&
     Object.keys(this.indexableFields).length > 0
 }
@@ -54,6 +55,8 @@ Search.prototype.init = function () {
   this.wordCollection = config.get('search.wordCollection')
   this.searchCollection = this.model.searchCollection || this.model.name + 'Search'
 
+  debug('initialised wordCollection: %s, indexCollection: %s', this.wordCollection, this.searchCollection)
+
   this.initialiseConnections()
   this.applyIndexListeners()
 }
@@ -113,17 +116,19 @@ Search.prototype.applyIndexListeners = function () {
  * @return {Promise} - resolves with a query containing IDs of documents that contain the searchTerm
  */
 Search.prototype.find = function (searchTerm) {
-  if (!this.canUse()) {
-    return {}
-  }
+  debug(this.canUse() ? 'search enabled' : 'search disabled')
+
+  debug('find in %s: %s', this.searchCollection, searchTerm)
 
   let tokenized = this.analyser.tokenize(searchTerm)
 
   return this.getWords(tokenized).then(words => {
-    words = words.results.map(word => word._id.toString())
+    let wordIds = words.results.map(word => word._id.toString())
+
+    debug('searching %s for words %o', this.searchCollection, words.results.map(word => word.word))
 
     return this.searchConnection.datastore.search({
-      words: words,
+      words: wordIds,
       collection: this.searchCollection,
       schema: this.getSearchSchema().fields,
       settings: this.getSearchSchema().settings,
@@ -152,6 +157,8 @@ Search.prototype.delete = function (docs) {
     return
   }
 
+  debug('deleting documents from the %s index: %o', this.searchCollection, docs.map(doc => doc._id.toString()))
+
   let deleteQueue = docs.map(doc => this.clearDocumentInstances(doc._id.toString()))
 
   return Promise.all(deleteQueue)
@@ -303,8 +310,6 @@ Search.prototype.indexDocument = function (document) {
 
     let data = this.formatInsertQuery(uniqueWords)
 
-    console.log(words, uniqueWords, data)
-
     if (!uniqueWords.length) {
       return this.clearAndInsertWordInstances(words, document._id.toString())
     }
@@ -317,27 +322,11 @@ Search.prototype.indexDocument = function (document) {
       schema: this.getWordSchema().fields,
       settings: this.getWordSchema().settings
     }).then(response => {
-      console.log('************')
-      console.log(response)
       return this.clearAndInsertWordInstances(words, document._id.toString())
+    }).catch(err => {
+      console.log(err)
     })
-  .catch(err => {
-    console.log(err)
-    // code `11000` returns if the word already exists, continue regardless
-    // MONGO SPECIFIC ERROR
-    // if (err.code === 11000) {
-    //   return this.clearAndInsertWordInstances(words, document._id.toString())
-    // }
   })
-  })
-
-  // return this.insert(
-  //   this.wordConnection.datastore,
-  //   data,
-  //   this.wordCollection,
-  //   this.getWordSchema().fields,
-  //   { ordered: false }
-  // )
 }
 
 /**
@@ -392,7 +381,7 @@ Search.prototype.clearAndInsertWordInstances = function (words, docId) {
     // Get all word instances from Analyser
     this.clearDocumentInstances(docId).then(response => {
       if (response.deletedCount) {
-        // console.log(`Cleared ${response.deletedCount} documents`)
+        debug('Removed %s documents from the %s index', response.deletedCount, this.searchCollection)
       }
 
       this.insertWordInstances(results.results, docId)
@@ -445,12 +434,8 @@ Search.prototype.insertWordInstances = function (words, docId) {
  * @return {Promise} - Database delete query.
  */
 Search.prototype.clearDocumentInstances = function (docId) {
-  let query = {
-    document: docId
-  }
-
   return this.searchConnection.datastore.delete({
-    query,
+    query: { document: docId },
     collection: this.searchCollection,
     schema: this.getSearchSchema().fields
   })
@@ -563,11 +548,11 @@ Search.prototype.getSearchSchema = function () {
   return {
     fields: {
       word: {
-        type: 'String',
+        type: 'Reference',
         required: true
       },
       document: {
-        type: 'String',
+        type: 'Reference',
         required: true
       },
       weight: {

From 8a39a3e8fbac0d4b311360ce8ae3398a8c6a6237 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 12 Jun 2018 13:18:57 +1200
Subject: [PATCH 10/33] test: update search unit tests; add search method to
 test connector

---
 README.md                      |   2 +-
 config/config.test.json.sample |   4 +-
 dadi/lib/search/index.js       |  34 +++++-----
 package.json                   |   2 +
 test/mocha.opts                |   1 -
 test/test-connector/index.js   |  57 +++++++++++++++--
 test/unit/search/index.js      | 110 +++++++++++++++++----------------
 7 files changed, 133 insertions(+), 77 deletions(-)

diff --git a/README.md b/README.md
index 6e7511db..46e5a9b5 100755
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 <img src="https://dadi.cloud/assets/products/dadi-api-full.png" alt="DADI API" height="65"/>
 
 [![npm (scoped)](https://img.shields.io/npm/v/@dadi/api.svg?maxAge=10800&style=flat-square)](https://www.npmjs.com/package/@dadi/api)
-[![coverage](https://img.shields.io/badge/coverage-82%25-yellow.svg?style=flat)](https://github.com/dadi/api)
+[![coverage](https://img.shields.io/badge/coverage-84%25-yellow.svg?style=flat)](https://github.com/dadi/api)
 [![Build Status](https://travis-ci.org/dadi/api.svg?branch=master)](https://travis-ci.org/dadi/api)
 [![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](http://standardjs.com/)
 
diff --git a/config/config.test.json.sample b/config/config.test.json.sample
index b33f7cfc..32286ff6 100755
--- a/config/config.test.json.sample
+++ b/config/config.test.json.sample
@@ -48,10 +48,10 @@
     "basePath": "test/acceptance/workspace/media"
   },
   "search": {
-    "enabled": true,
+    "enabled": false,
     "minQueryLength": 3,
     "wordCollection": "words",
-    "datastore": "@dadi/api-mongodb",
+    "datastore": "./../../../test/test-connector",
     "database": "search"
   },
   "feedback": false,
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index 256d55b7..2c3460d2 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -5,6 +5,7 @@ const config = require(path.join(__dirname, '/../../../config'))
 const Connection = require(path.join(__dirname, '/../model/connection'))
 const debug = require('debug')('api:search')
 const DataStore = require(path.join(__dirname, '../datastore'))
+const promiseQueue = require('js-promise-queue')
 const StandardAnalyser = require('./analysers/standard')
 const DefaultAnalyser = StandardAnalyser
 const pageLimit = 20
@@ -145,21 +146,21 @@ Search.prototype.find = function (searchTerm) {
 
 /**
  * Removes entries in the collection's search collection that match the specified documents
- * @param {Array} docs - an array of documents for which to remove word instances
+ * @param {Array} documents - an array of documents for which to remove word instances
  * @return {Promise} - Query to delete instances with matching document ids.
  */
-Search.prototype.delete = function (docs) {
+Search.prototype.delete = function (documents) {
   if (!this.canUse()) {
     return Promise.resolve()
   }
 
-  if (!Array.isArray(docs)) {
+  if (!Array.isArray(documents)) {
     return
   }
 
-  debug('deleting documents from the %s index: %o', this.searchCollection, docs.map(doc => doc._id.toString()))
+  debug('deleting documents from the %s index', this.searchCollection)
 
-  let deleteQueue = docs.map(doc => this.clearDocumentInstances(doc._id.toString()))
+  let deleteQueue = documents.map(document => this.clearDocumentInstances(document._id.toString()))
 
   return Promise.all(deleteQueue)
 }
@@ -262,28 +263,32 @@ Search.prototype.hasSearchField = function (field) {
 /**
  * Removes properties from the specified document that aren't configured to be indexed
  *
- * @param  {Object} doc - a document to be indexed
+ * @param  {Object} document - a document to be indexed
  * @return {Object} the specified document with non-indexable properties removed
  */
-Search.prototype.removeNonIndexableFields = function (doc) {
-  if (typeof doc !== 'object') return {}
+Search.prototype.removeNonIndexableFields = function (document) {
+  if (typeof document !== 'object') return {}
 
-  return Object.assign({}, ...Object.keys(doc)
+  return Object.assign({}, ...Object.keys(document)
     .filter(key => this.indexableFields[key])
     .map(key => {
-      return {[key]: doc[key]}
+      return {[key]: document[key]}
     }))
 }
 
 /**
  * Index the specified documents
- * @param {Array} docs - an array of documents to be indexed
+ * @param {Array} documents - an array of documents to be indexed
  * @return {Promise} - Queries to index documents.
  */
-Search.prototype.index = function (docs) {
-  if (!this.canUse() || !Array.isArray(docs)) return Promise.resolve()
+Search.prototype.index = function (documents) {
+  if (!this.canUse() || !Array.isArray(documents)) {
+    return Promise.resolve()
+  }
 
-  return Promise.all(docs.map(doc => this.indexDocument(doc)))
+  promiseQueue(documents, this.indexDocument.bind(this), {
+    interval: 300
+  })
 }
 
 /**
@@ -294,7 +299,6 @@ Search.prototype.index = function (docs) {
  * @return {[type]}     [description]
  */
 Search.prototype.indexDocument = function (document) {
-  // let analyser = new DefaultAnalyser(this.indexableFields)
   let reducedDocument = this.removeNonIndexableFields(document)
   let words = this.analyseDocumentWords(reducedDocument)
   let uniqueWords
diff --git a/package.json b/package.json
index 6c7c4c13..32095f78 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
     "deepmerge": "^2.1.0",
     "fs-extra": "^3.0.1",
     "imagesize": "^1.0.0",
+    "js-promise-queue": "^1.1.0",
     "jsonwebtoken": "^8.0.0",
     "length-stream": "^0.1.1",
     "mime": "^2.3.1",
@@ -65,6 +66,7 @@
     "@dadi/metadata": "^2.0.0",
     "aws-sdk-mock": "1.6.1",
     "env-test": "1.0.0",
+    "faker": "^4.1.0",
     "fakeredis": "1.0.3",
     "form-data": "2.1.4",
     "husky": "^0.13.3",
diff --git a/test/mocha.opts b/test/mocha.opts
index 109da9bd..484868ef 100755
--- a/test/mocha.opts
+++ b/test/mocha.opts
@@ -1,4 +1,3 @@
---bail
 --full-trace
 --timeout=4000
 --ui bdd
diff --git a/test/test-connector/index.js b/test/test-connector/index.js
index d2bb08aa..69cdaae6 100644
--- a/test/test-connector/index.js
+++ b/test/test-connector/index.js
@@ -121,7 +121,7 @@ DataStore.prototype.connect = function ({database, collection}) {
     })
 
     if (DEBUG) {
-      this.database.___id = Math.random()  
+      this.database.___id = Math.random()
     }
 
     databasePool[connectionKey] = this.database
@@ -162,9 +162,8 @@ DataStore.prototype.delete = function ({query, collection, schema}) {
       if (DEBUG) {
         console.log('')
         console.log('* (Test connector) Delete:', this.database.___id, query, results.data(), count)
-        console.log('')  
+        console.log('')
       }
-      
 
       results.remove()
 
@@ -469,6 +468,56 @@ DataStore.prototype.insert = function ({data, collection, options = {}, schema,
   })
 }
 
+/** Search for documents in the database
+ *
+ * @param {Object|Array} words -
+ * @param {string} collection - the name of the collection to search
+ * @param {object} options - options to modify the query
+ * @param {Object} schema - the JSON schema for the collection
+ * @param {Object} settings - the JSON settings configuration for the collection
+ * @returns {Promise.<Array, Error>}
+ */
+DataStore.prototype.search = function ({ words, collection, options = {}, schema, settings }) {
+  if (this._mockIsDisconnected(collection)) {
+    this.readyState = STATE_DISCONNECTED
+
+    return Promise.reject(new Error('DB_DISCONNECTED'))
+  }
+
+  debug('search in %s for %o', collection, words)
+
+  let query = [
+    {
+      $match: {
+        word: {
+          $in: words
+        }
+      }
+    },
+    {
+      $group: {
+        _id: { document: '$document' },
+        count: { $sum: 1 },
+        weight: { $sum: '$weight' }
+      }
+    },
+    {
+      $sort: {
+        weight: -1
+      }
+    },
+    { $limit: options.limit || 100 }
+  ]
+
+  return this.find({
+    query,
+    collection,
+    options,
+    schema,
+    settings
+  })
+}
+
 /**
  *
  */
@@ -608,4 +657,4 @@ module.exports._mock = {
   setFailedConnectionExceptions: collections => {
     mockConnection.exceptForCollections = collections
   }
-}
\ No newline at end of file
+}
diff --git a/test/unit/search/index.js b/test/unit/search/index.js
index af44e24f..8903d201 100644
--- a/test/unit/search/index.js
+++ b/test/unit/search/index.js
@@ -1,8 +1,9 @@
-const acceptanceHelper = require(__dirname + '/../../acceptance/help')
-const config = require(__dirname + '/../../../config')
-const help = require(__dirname + '/../help')
-const Model = require(__dirname + '/../../../dadi/lib/model')
-const Search = require(__dirname + '/../../../dadi/lib/search')
+const acceptanceHelper = require('./../../acceptance/help')
+const config = require('./../../../config')
+const faker = require('faker')
+const help = require('./../help')
+const Model = require('./../../../dadi/lib/model')
+const Search = require('./../../../dadi/lib/search')
 const should = require('should')
 const sinon = require('sinon')
 const store = require(config.get('search.datastore'))
@@ -10,7 +11,11 @@ const store = require(config.get('search.datastore'))
 let mod
 let searchInstance
 
-describe.skip('Search', () => {
+describe('Search', () => {
+  before(() => {
+    config.set('search.enabled', true)
+  })
+
   beforeEach(done => {
     mod = Model('testSearchModel', help.getSearchModelSchema(), null, { database: 'testdb' })
     searchInstance = new Search(mod)
@@ -18,6 +23,10 @@ describe.skip('Search', () => {
     done()
   })
 
+  after(() => {
+    config.set('search.enabled', false)
+  })
+
   it('should export constructor', done => {
     Search.should.be.Function
     done()
@@ -40,17 +49,12 @@ describe.skip('Search', () => {
       setTimeout(() => {
         should.exist(searchInstance.wordConnection.db)
         should.exist(searchInstance.searchConnection.db)
-        searchInstance.wordConnection.db.config.hosts[0].host.should.eql('127.0.0.1')
-        searchInstance.wordConnection.db.config.hosts[0].port.should.eql(27017)
-        searchInstance.searchConnection.db.config.hosts[0].host.should.eql('127.0.0.1')
-        searchInstance.searchConnection.db.config.hosts[0].port.should.eql(27017)
-
         done()
       }, 500)
     })
   })
 
-  describe('`applyIndexListeners` method', () => {
+  describe.skip('`applyIndexListeners` method', () => {
     it('should call database index method once connection is established', done => {
       mod = Model('testModelNew', help.getSearchModelSchema(), null, { database: 'testdb' })
       const dbIndexStub = sinon.spy(store.prototype, 'index')
@@ -115,10 +119,10 @@ describe.skip('Search', () => {
     })
   })
 
-  describe('`createWordInstanceInsertQuery` method', () => {
+  describe('`formatInsertQuery` method', () => {
     it('should convert list of words to valid insert query object', done => {
-      searchInstance.createWordInstanceInsertQuery(['foo']).should.be.an.instanceOf(Array)
-      searchInstance.createWordInstanceInsertQuery(['foo'])[0].should.have.property('word', 'foo')
+      searchInstance.formatInsertQuery(['foo']).should.be.an.instanceOf(Array)
+      searchInstance.formatInsertQuery(['foo'])[0].should.have.property('word', 'foo')
       done()
     })
   })
@@ -140,7 +144,7 @@ describe.skip('Search', () => {
     })
   })
 
-  describe('`runFind` method', () => {
+  describe.skip('`runFind` method', () => {
     it('should search the database based on the query', done => {
       const dbFindStub = sinon.spy(store.prototype, 'find')
 
@@ -189,7 +193,7 @@ describe.skip('Search', () => {
     })
   })
 
-  describe('`insert` method', () => {
+  describe.skip('`insert` method', () => {
     it('should not execute the database insert if no data is provided', done => {
       const dbInsertStub = sinon.spy(store.prototype, 'insert')
 
@@ -267,27 +271,27 @@ describe.skip('Search', () => {
     it('should call runBatchIndex repeatedly when there are more results', done => {
       let schema = help.getSearchModelSchema()
       let mod = Model('testSearchModel', schema, null, { database: 'testdb' })
-      const indexable = new Search(mod)
+      let indexable = new Search(mod)
       indexable.init()
 
-      const spy = sinon.spy(indexable, 'runBatchIndex')
+      let spy = sinon.spy(indexable, 'runBatchIndex')
 
-      function guid () {
-        return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
-          var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8)
-          return v.toString(16)
-        })
-      }
-
-      var docs = [
-        { searchableFieldName: guid() },
-        { searchableFieldName: guid() },
-        { searchableFieldName: guid() },
-        { searchableFieldName: guid() },
-        { searchableFieldName: guid() }
+      let docs = [
+        { searchableFieldName: faker.name.findName() },
+        { searchableFieldName: faker.name.findName() },
+        { searchableFieldName: faker.name.findName() },
+        { searchableFieldName: faker.name.findName() },
+        { searchableFieldName: faker.name.findName() }
       ]
 
-      const indexStub = sinon.stub(indexable, 'index').callsFake(() => {
+      // insert documents directly
+      mod.connection.db.insert({
+        data: docs,
+        collection: 'testSearchModel',
+        schema
+      })
+
+      let indexStub = sinon.stub(indexable, 'index').callsFake(() => {
         return Promise.resolve({
           results: docs,
           metadata: {
@@ -297,27 +301,25 @@ describe.skip('Search', () => {
         })
       })
 
-      mod.create(docs, {}, obj => {
-        indexable.batchIndex(1, 1)
-
-        setTimeout(() => {
-          spy.restore()
-          indexStub.restore()
-          spy.callCount.should.be.above(1)
-          let args = spy.args
-          args[0][0].skip.should.eql(0)
-          args[0][0].page.should.eql(1)
-          args[1][0].skip.should.eql(1)
-          args[1][0].page.should.eql(2)
-          args[2][0].skip.should.eql(2)
-          args[2][0].page.should.eql(3)
-          args[3][0].skip.should.eql(3)
-          args[3][0].page.should.eql(4)
-          args[4][0].skip.should.eql(4)
-          args[4][0].page.should.eql(5)
-          done()
-        }, 3000)
-      })
+      indexable.batchIndex(1, 1)
+
+      setTimeout(() => {
+        spy.restore()
+        indexStub.restore()
+        spy.callCount.should.be.above(1)
+        let args = spy.args
+        args[0][0].skip.should.eql(0)
+        args[0][0].page.should.eql(1)
+        args[1][0].skip.should.eql(1)
+        args[1][0].page.should.eql(2)
+        args[2][0].skip.should.eql(2)
+        args[2][0].page.should.eql(3)
+        args[3][0].skip.should.eql(3)
+        args[3][0].page.should.eql(4)
+        args[4][0].skip.should.eql(4)
+        args[4][0].page.should.eql(5)
+        done()
+      }, 3000)
     })
   })
 })

From 57588e87282bc744217f11d40374d541b815caaa Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 17 Jul 2018 13:28:54 +0300
Subject: [PATCH 11/33] test: improve acceptance tests with latest search
 integration

---
 dadi/lib/controller/documents.js              |   2 +-
 dadi/lib/controller/index.js                  |   2 +-
 dadi/lib/index.js                             |   4 -
 dadi/lib/model/search.js                      |   2 +-
 dadi/lib/search/index.js                      |  35 ++--
 test/acceptance/cache.js                      |  72 -------
 test/acceptance/help.js                       |   8 +-
 test/acceptance/search.js                     | 179 +++++++++++-------
 .../vtest/testdb/collection.test-schema.json  |   7 +
 test/mocha.opts                               |   1 +
 test/test-connector/index.js                  |  79 +++++---
 test/unit/storage.s3.js                       |   4 +-
 12 files changed, 197 insertions(+), 198 deletions(-)

diff --git a/dadi/lib/controller/documents.js b/dadi/lib/controller/documents.js
index 289fc833..2c867faa 100644
--- a/dadi/lib/controller/documents.js
+++ b/dadi/lib/controller/documents.js
@@ -214,7 +214,7 @@ Collection.prototype.registerRoutes = function (route, filePath) {
   })
 
   // Creating generic route.
-  this.server.app.use(`${route}/:id(${this.ID_PATTERN})?/:action(count|stats)?`, (req, res, next) => {
+  this.server.app.use(`${route}/:id(${this.ID_PATTERN})?/:action(count|search|stats)?`, (req, res, next) => {
     try {
       // Map request method to controller method.
       let method = req.params.action || (req.method && req.method.toLowerCase())
diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index f2dfaf09..e8a92b78 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -181,7 +181,7 @@ Controller.prototype.search = function (req, res, next) {
   let queryOptions = this._prepareQueryOptions(options)
 
   if (queryOptions.errors.length !== 0) {
-    sendBackJSON(400, res, next)(null, queryOptions)
+    return help.sendBackJSON(400, res, next)(null, queryOptions)
   } else {
     queryOptions = queryOptions.queryOptions
   }
diff --git a/dadi/lib/index.js b/dadi/lib/index.js
index 4dd44257..1269fe85 100755
--- a/dadi/lib/index.js
+++ b/dadi/lib/index.js
@@ -35,7 +35,6 @@ var help = require(path.join(__dirname, '/help'))
 var Model = require(path.join(__dirname, '/model'))
 var mediaModel = require(path.join(__dirname, '/model/media'))
 var monitor = require(path.join(__dirname, '/monitor'))
-var search = require(path.join(__dirname, '/search'))
 
 var config = require(path.join(__dirname, '/../../config'))
 
@@ -247,9 +246,6 @@ Server.prototype.start = function (done) {
   // caching layer
   cache(this).init()
 
-  // search layer
-  search(this)
-
   // start listening
   var server = this.server = app.listen()
 
diff --git a/dadi/lib/model/search.js b/dadi/lib/model/search.js
index a83128e1..8317649f 100644
--- a/dadi/lib/model/search.js
+++ b/dadi/lib/model/search.js
@@ -2,7 +2,7 @@ const config = require('./../../../config')
 const debug = require('debug')('api:model:search')
 
 /**
- * Searchs for documents in the datbase and returns a
+ * Searches for documents in the database and returns a
  * metadata object.
  *
  * @param {Object} query - the search query
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index 2c3460d2..12d64060 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -71,8 +71,8 @@ Search.prototype.initialiseConnections = function () {
 
   this.wordConnection = Connection(
     {
-      database: searchConfig.database,
       collection: this.wordCollection,
+      database: searchConfig.database,
       override: true
     },
     this.wordCollection,
@@ -81,8 +81,8 @@ Search.prototype.initialiseConnections = function () {
 
   this.searchConnection = Connection(
     {
-      database: searchConfig.database,
       collection: this.searchCollection,
+      database: searchConfig.database,
       override: true
     },
     this.searchCollection,
@@ -135,11 +135,15 @@ Search.prototype.find = function (searchTerm) {
       settings: this.getSearchSchema().settings,
       opions: { limit: pageLimit }
     }).then(wordInstances => {
+      wordInstances = wordInstances.map(instance => instance._id.document)
+
       return {
         _id: {
-          '$containsAny': wordInstances.map(instance => instance._id.document)
+          '$containsAny': wordInstances
         }
       }
+    }).catch(err => {
+      console.log(err)
     })
   })
 }
@@ -183,7 +187,7 @@ Search.prototype.getWords = function (words) {
     settings: this.getWordSchema().settings
   }).then(response => {
     // Try a second pass with regular expressions
-    if (!response.length) {
+    if (response.results.length === 0) {
       let regexWords = words.map(word => new RegExp(word))
       let regexQuery = { word: { '$containsAny': regexWords } }
 
@@ -445,22 +449,6 @@ Search.prototype.clearDocumentInstances = function (docId) {
   })
 }
 
-/**
- * Insert documents into the database
- *
- * @param {Connection} database - the database connection
- * @param {Object|Array} data - the data to insert into the database
- * @param {String} collection - the name of the collection to insert into
- * @param {Object} schema - the collection schema
- * @param {Object} options - options to use in the query
- * @return {Promise}
- */
-// Search.prototype.insert = function (datastore, data, collection, schema, options = {}) {
-//   console.log(this.datastore)
-//   if (!data.length) return Promise.resolve()
-//   return datastore.insert({data, collection, options, schema})
-// }
-
 /**
  * Index an entire collection, in batches of documents
  *
@@ -471,7 +459,6 @@ Search.prototype.batchIndex = function (page = 1, limit = 1000) {
   if (!Object.keys(this.indexableFields).length) return
 
   let skip = (page - 1) * limit
-  console.log(`Indexing page ${page} (${limit} per page)`)
 
   let fields = Object.assign({}, ...Object.keys(this.indexableFields).map(key => {
     return {[key]: 1}
@@ -484,6 +471,8 @@ Search.prototype.batchIndex = function (page = 1, limit = 1000) {
     fields
   }
 
+  debug(`Indexing page ${page} (${limit} per page)`)
+
   if (this.model.connection.db) {
     this.runBatchIndex(options)
   }
@@ -507,11 +496,11 @@ Search.prototype.runBatchIndex = function (options) {
     settings: this.model.settings
   }).then(results => {
     if (results.results && results.results.length) {
-      console.log(`Indexed ${results.results.length} ${results.results.length === 1 ? 'record' : 'records'} for ${this.model.name}`)
+      debug(`Indexed ${results.results.length} ${results.results.length === 1 ? 'record' : 'records'} for ${this.model.name}`)
 
       if (results.results.length > 0) {
         this.index(results.results).then(response => {
-          console.log(`Indexed page ${options.page}/${results.metadata.totalPages}`)
+          debug(`Indexed page ${options.page}/${results.metadata.totalPages}`)
 
           if (options.page * options.limit < results.metadata.totalCount) {
             return this.batchIndex(options.page + 1, options.limit)
diff --git a/test/acceptance/cache.js b/test/acceptance/cache.js
index 1e439b0e..61f3e1e6 100755
--- a/test/acceptance/cache.js
+++ b/test/acceptance/cache.js
@@ -571,78 +571,6 @@ describe('Cache', function (done) {
       done()
     })
 
-    it.skip('should throw error if can\'t connect to Redis client', function (done) {
-      delete require.cache[__dirname + '/../../config.js']
-      delete require.cache[__dirname + '/../../dadi/lib/']
-
-      config.loadFile(config.configPath())
-
-      should.throws(function () { app.start(function () {}) }, Error)
-
-      app.stop(done)
-    })
-
-    it.skip('should initialise Redis client', function (done) {
-      delete require.cache[__dirname + '/../../config.js']
-      config.loadFile(config.configPath())
-
-      // sinon.stub(redis, 'createClient', fakeredis.createClient);
-
-      delete require.cache[__dirname + '/../../dadi/lib/']
-      cache.reset()
-
-      try {
-        app.stop(function () {})
-        // app.start(function(){});
-      } catch (err) {
-      }
-
-      var c = cache(app)
-      // redis.createClient.restore();
-      // c.redisClient.should.not.be.null;
-      // app.stop(function(){});
-      done()
-    })
-
-    it.skip('should fallback to directory cache if Redis client fails', function (done) {
-      delete require.cache[__dirname + '/../../config.js']
-      config.loadFile(config.configPath())
-
-      var EventEmitter = require('events')
-      var util = require('util')
-
-      /* Fake redis client */
-      function Client () {
-        this.end = function (reallyEnd) { }
-        EventEmitter.call(this)
-      }
-
-      util.inherits(Client, EventEmitter)
-      var redisClient = new Client()
-      /* End Fake redis client */
-
-      sinon.stub(redis, 'createClient').returns(redisClient)
-
-      delete require.cache[__dirname + '/../../dadi/lib/']
-      cache.reset()
-
-      var c = cache(app)
-      // redis.createClient.restore();
-
-      setTimeout(function () {
-        // emit an error event
-        redisClient.emit('error', { code: 'CONNECTION_BROKEN'})
-
-        config.get('caching.directory.enabled').should.eql(true)
-
-        try {
-          app.stop(done)
-        } catch (err) {
-          done()
-        }
-      }, 1000)
-    })
-
     it('should check key exists in Redis', function (done) {
       delete require.cache[__dirname + '/../../dadi/lib/']
 
diff --git a/test/acceptance/help.js b/test/acceptance/help.js
index 50e3649f..a00c2111 100755
--- a/test/acceptance/help.js
+++ b/test/acceptance/help.js
@@ -15,7 +15,7 @@ module.exports.createDoc = function (token, done) {
     .post('/vtest/testdb/test-schema')
     .set('Authorization', 'Bearer ' + token)
     .send({field1: ((Math.random() * 10) | 1).toString()})
-    //.expect(200)
+    // .expect(200)
     .end(function (err, res) {
       if (err) return done(err)
       res.body.results.length.should.equal(1)
@@ -423,7 +423,7 @@ module.exports.getCollectionMap = function () {
     databases.forEach(database => {
       let databasePath = path.join(versionPath, database)
       let stats = fs.statSync(databasePath)
-      
+
       if (stats.isDirectory()) {
         let collections = fs.readdirSync(databasePath)
 
@@ -439,7 +439,7 @@ module.exports.getCollectionMap = function () {
 
           map[`/${version}/${database}/${collectionName}`] = require(collectionPath)
         })
-      }      
+      }
     })
   })
 
@@ -469,7 +469,7 @@ module.exports.writeTempFile = function (filePath, data, callback) {
   fs.ensureDir(
     path.dirname(fullPath),
     err => {
-      fs.writeFileSync(fullPath, parsedData)    
+      fs.writeFileSync(fullPath, parsedData)
     }
   )
 
diff --git a/test/acceptance/search.js b/test/acceptance/search.js
index cf564b72..a713c77e 100644
--- a/test/acceptance/search.js
+++ b/test/acceptance/search.js
@@ -9,54 +9,85 @@ var connection = require(__dirname + '/../../dadi/lib/model/connection')
 var config = require(__dirname + '/../../config')
 var help = require(__dirname + '/help')
 var app = require(__dirname + '/../../dadi/lib/')
+var model = require(__dirname + '/../../dadi/lib/model/')
 
 // variables scoped for use throughout tests
 var bearerToken
 var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
 var lastModifiedAt = 0
 
-describe.skip('Search', function () {
+describe('Search', function () {
   this.timeout(4000)
 
+  let cleanupFn
+
   before(function (done) {
-    app.start(() => {
-      help.dropDatabase('testdb', function (err) {
-        if (err) return done(err)
+    help.dropDatabase('testdb', function (err) {
+      if (err) return done(err)
+
+      config.set('search', {
+        'enabled': true,
+        'minQueryLength': 3,
+        'wordCollection': 'words',
+        'datastore': './../../../test/test-connector',
+        'database': 'testdb'
+      })
 
+      app.start(function () {
         help.getBearerTokenWithAccessType('admin', function (err, token) {
           if (err) return done(err)
 
           bearerToken = token
 
-          // add a searchable field to the schema
-          var jsSchemaString = fs.readFileSync(__dirname + '/../new-schema.json', {encoding: 'utf8'})
-          jsSchemaString = jsSchemaString.replace('newField', 'field1')
-          var schema = JSON.parse(jsSchemaString)
-
-          schema.fields.title = _.extend({}, schema.fields.newField, {
-            type: 'String',
-            required: false,
-            search: {
-              weight: 2
+          let schema = {
+            'fields': {
+              'field1': {
+                'type': 'String',
+                'required': false
+              },
+              'title': {
+                'type': 'String',
+                'required': false,
+                'search': {
+                  'weight': 2
+                }
+              },
+              'field2': {
+                'type': 'Number',
+                'required': false
+              },
+              'field3': {
+                'type': 'ObjectID',
+                'required': false
+              },
+              '_fieldWithUnderscore': {
+                'type': 'Object',
+                'required': false
+              }
+            },
+            'settings': {
+              'count': 40
             }
-          })
-
-          var client = request(connectionString)
-
-          client
-            .post('/vtest/testdb/test-schema/config')
-            .send(JSON.stringify(schema, null, 2))
-            .set('content-type', 'text/plain')
-            .set('Authorization', 'Bearer ' + bearerToken)
-            .expect('content-type', 'application/json')
-            .end((err, res) => {
-              if (err) return done(err)
-
-              // let's wait a bit
-              setTimeout(function () {
-                done()
-              }, 500)
-            })
+          }
+
+          help.writeTempFile(
+            'temp-workspace/collections/vtest/testdb/collection.test-schema.json',
+            schema,
+            callback1 => {
+              help.writeTempFile(
+                'temp-workspace/collections/v1/testdb/collection.test-schema.json',
+                schema,
+                callback2 => {
+                  cleanupFn = () => {
+                    callback1()
+                    callback2()
+                  }
+
+                  done()
+                }
+              )
+            }
+          )
         })
       })
     })
@@ -67,49 +98,32 @@ describe.skip('Search', function () {
       'enabled': false
     })
 
-    var cleanup = function (done) {
-      // try {
-      //   fs.unlinkSync(config.get('paths').collections + '/vtest/testdb/collection.test-schema.json')
-      // } catch (e) {}
-
+    app.stop(() => {
+      cleanupFn()
       done()
-    }
-
-    help.removeTestClients(() => {
-      app.stop(() => {
-        setTimeout(() => {
-          cleanup(done)
-        }, 500)
-      })
     })
   })
 
   describe('Disabled', function () {
     it('should return 501 when calling a /search endpoint', function (done) {
-      config.set('search', {
-        'enabled': false,
-        'minQueryLength': 3,
-        'datastore': '@dadi/api-mongodb',
-        'database': 'search'
-      })
+      config.set('search.enabled', false)
 
       var client = request(connectionString)
       client
       .get('/vtest/testdb/test-schema/search')
       .set('Authorization', 'Bearer ' + bearerToken)
       .expect(501)
-      .end(done)
+      .end((err, res) => {
+        config.set('search.enabled', true)
+        done()
+      })
     })
   })
 
   describe('Enabled', function () {
     it('should return 400 when calling a /search endpoint with no query', function (done) {
-      config.set('search', {
-        'enabled': true,
-        'minQueryLength': 3,
-        'datastore': '@dadi/api-mongodb',
-        'database': 'search'
-      })
+      let searchModel = model('test-schema')
+      searchModel.searchHandler.init()
 
       var client = request(connectionString)
       client
@@ -120,12 +134,8 @@ describe.skip('Search', function () {
     })
 
     it('should return 400 when calling a /search endpoint with a short query', function (done) {
-      config.set('search', {
-        'enabled': true,
-        'minQueryLength': 3,
-        'datastore': '@dadi/api-mongodb',
-        'database': 'search'
-      })
+      let searchModel = model('test-schema')
+      searchModel.searchHandler.init()
 
       var client = request(connectionString)
       client
@@ -139,6 +149,9 @@ describe.skip('Search', function () {
     })
 
     it('should return empty results when no documents match a query', function (done) {
+      let searchModel = model('test-schema')
+      searchModel.searchHandler.init()
+
       var client = request(connectionString)
       client
       .get('/vtest/testdb/test-schema/search?q=xxx')
@@ -154,6 +167,9 @@ describe.skip('Search', function () {
     })
 
     it('should return results when documents match a query', function (done) {
+      let searchModel = model('test-schema')
+      searchModel.searchHandler.init()
+
       var client = request(connectionString)
 
       var doc = {
@@ -175,8 +191,43 @@ describe.skip('Search', function () {
         .end((err, res) => {
           if (err) return done(err)
           should.exist(res.body.results)
+
           res.body.results.should.be.Array
           res.body.results.length.should.eql(1)
+
+          done()
+        })
+      })
+    })
+
+    it('should return metadata containing the search term', function (done) {
+      let searchModel = model('test-schema')
+      searchModel.searchHandler.init()
+
+      var client = request(connectionString)
+
+      var doc = {
+        field1: 'The quick brown fox jumps',
+        title: 'The quick brown fox jumps over the lazy dog'
+      }
+
+      client
+      .post('/vtest/testdb/test-schema')
+      .set('Authorization', 'Bearer ' + bearerToken)
+      .set('content-type', 'application/json')
+      .send(doc)
+      .expect(200)
+      .end((err, res) => {
+        client
+        .get('/vtest/testdb/test-schema/search?q=quick%20brown')
+        .set('Authorization', 'Bearer ' + bearerToken)
+        .expect(200)
+        .end((err, res) => {
+          if (err) return done(err)
+          should.exist(res.body.metadata)
+          should.exist(res.body.metadata.search)
+          res.body.metadata.search.should.eql('quick brown')
+
           done()
         })
       })
diff --git a/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json b/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
index 66de1621..c59f8aba 100755
--- a/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
+++ b/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
@@ -6,6 +6,13 @@
       "comments": "The title of the entry",
       "validation": {},
       "required": false
+    },
+    "title": {
+      "type": "String",
+      "label": "Title",
+      "comments": "The title of the entry",
+      "validation": {},
+      "required": false
     }
   },
   "settings": {
diff --git a/test/mocha.opts b/test/mocha.opts
index 484868ef..109da9bd 100755
--- a/test/mocha.opts
+++ b/test/mocha.opts
@@ -1,3 +1,4 @@
+--bail
 --full-trace
 --timeout=4000
 --ui bdd
diff --git a/test/test-connector/index.js b/test/test-connector/index.js
index 089fa50d..e6244a70 100644
--- a/test/test-connector/index.js
+++ b/test/test-connector/index.js
@@ -287,7 +287,7 @@ DataStore.prototype.find = function ({ query, collection, options = {}, schema,
         collection: collName,
         query,
         results
-      })      
+      })
 
       let returnData = {}
       returnData.results = results.map(this.formatDocumentForOutput.bind(this))
@@ -520,35 +520,62 @@ DataStore.prototype.search = function ({ words, collection, options = {}, schema
 
   debug('search in %s for %o', collection, words)
 
-  let query = [
-    {
-      $match: {
+  function mapFn (document) {
+    return {
+      document: document.document,
+      word: document.word,
+      weight: document.weight
+    }
+  }
+
+  function reduceFn (documents) {
+    let matches = documents.reduce((groups, document) => {
+      let key = document.document
+
+      groups[key] = groups[key] || {
+        count: 0,
+        weight: 0
+      }
+
+      groups[key].count++
+      groups[key].weight = groups[key].weight + document.weight
+      return groups
+    }, {})
+
+    let output = []
+
+    Object.keys(matches).forEach(function (match) {
+      output.push({
+        _id: {
+          document: match
+        },
+        count: matches[match].count,
+        weight: matches[match].weight
+      })
+    })
+
+    output.sort(function (a, b) {
+      if (a.weight === b.weight) return 0
+      return a.weight < b.weight ? 1 : -1
+    })
+
+    return output
+  }
+
+  return new Promise((resolve, reject) => {
+    this.getCollection(collection).then(collection => {
+      let results
+
+      let query = {
         word: {
-          $in: words
+          '$containsAny': words
         }
       }
-    },
-    {
-      $group: {
-        _id: { document: '$document' },
-        count: { $sum: 1 },
-        weight: { $sum: '$weight' }
-      }
-    },
-    {
-      $sort: {
-        weight: -1
-      }
-    },
-    { $limit: options.limit || 100 }
-  ]
 
-  return this.find({
-    query,
-    collection,
-    options,
-    schema,
-    settings
+      let baseResultset = collection.chain().find(query)
+
+      return resolve(baseResultset.mapReduce(mapFn, reduceFn))
+    })
   })
 }
 
diff --git a/test/unit/storage.s3.js b/test/unit/storage.s3.js
index d31da43a..72356f8e 100644
--- a/test/unit/storage.s3.js
+++ b/test/unit/storage.s3.js
@@ -85,7 +85,7 @@ describe('Storage', function (done) {
       })
     })
 
-    it.skip('should call S3 API with the correct parameters when deleting media', function (done) {
+    it('should call S3 API with the correct parameters when deleting media', function (done) {
       config.set('media.enabled', true)
       config.set('media.storage', 's3')
       config.set('media.s3.bucketName', 'testbucket')
@@ -118,7 +118,7 @@ describe('Storage', function (done) {
       })
     })
 
-    it.skip('should call S3 API with the correct parameters when requesting media', function (done) {
+    it('should call S3 API with the correct parameters when requesting media', function (done) {
       config.set('media.enabled', true)
       config.set('media.storage', 's3')
       config.set('media.s3.bucketName', 'testbucket')

From 625737045e8428f1b4519ead10a751738d163f7e Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 17 Jul 2018 13:37:13 +0300
Subject: [PATCH 12/33] fix: update config schema docs

---
 config.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/config.js b/config.js
index 4bd12b9d..fadbfdb2 100755
--- a/config.js
+++ b/config.js
@@ -145,19 +145,19 @@ var conf = convict({
       default: 3
     },
     wordCollection: {
-      doc: '',
+      doc: 'The name of the datastore collection that will hold tokenized words',
       format: String,
       default: 'words'
     },
     datastore: {
-      doc: "",
+      doc: 'The datastore to use for storing and querying indexed documents',
       format: String,
       default: '@dadi/api-mongodb'
     },
     database: {
-      doc: '',
+      doc: 'The name of the database to use for storing and querying indexed documents',
       format: String,
-      default: 'test',
+      default: 'search',
       env: 'DB_SEARCH_NAME'
     }
   },

From 5338d85563a191a6a097af4cae484c9078b655a6 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 17 Jul 2018 13:47:53 +0300
Subject: [PATCH 13/33] refactor: remove unnecessary catch

---
 dadi/lib/controller/index.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index e8a92b78..f400b382 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -204,8 +204,6 @@ Controller.prototype.search = function (req, res, next) {
       })
 
       return help.sendBackJSON(200, res, next)(null, results)
-    }).catch(error => {
-      return next(error)
     })
   }).catch(error => {
     return next(error)

From e77a651f4529495e1924e3f45bfe44496a96d5dc Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 17 Jul 2018 13:51:06 +0300
Subject: [PATCH 14/33] chore(package): update logger dependency

---
 package.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 64031fe4..35b381ad 100644
--- a/package.json
+++ b/package.json
@@ -18,12 +18,11 @@
     ]
   },
   "dependencies": {
-    "@dadi/api-mongodb": "^4.1.0",
     "@dadi/boot": "^1.1.3",
     "@dadi/cache": "^3.0.0",
     "@dadi/et": "^2.0.0",
     "@dadi/format-error": "^1.7.0",
-    "@dadi/logger": "^1.3.0",
+    "@dadi/logger": "^1.4.0",
     "@dadi/status": "latest",
     "async": "^2.6.0",
     "aws-sdk": "2.249.1",

From ebd5e67284cd493fc5bb3252cb1db3b28c935d85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Tue, 17 Jul 2018 13:40:34 +0100
Subject: [PATCH 15/33] fix: fix DELETE roles permissions

---
 dadi/lib/controller/clients.js                  | 7 +++----
 test/acceptance/acl/clients-api/roles-delete.js | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dadi/lib/controller/clients.js b/dadi/lib/controller/clients.js
index 26955928..7380d3e0 100644
--- a/dadi/lib/controller/clients.js
+++ b/dadi/lib/controller/clients.js
@@ -104,9 +104,8 @@ Clients.prototype.deleteRole = function (req, res, next) {
     // they have the role they are trying to remove.
     if (!model.isAdmin(req.dadiApiClient)) {
       return model.get(req.dadiApiClient.clientId).then(({results}) => {
-        let requestingClientHasRole = Boolean(
-          results.find(dbRole => dbRole.name === role)
-        )
+        let user = results[0]
+        let requestingClientHasRole = user.roles && user.roles.includes(role)
 
         return requestingClientHasRole
       })
@@ -129,7 +128,7 @@ Clients.prototype.deleteRole = function (req, res, next) {
       return help.sendBackJSON(404, res, next)(null)
     }
 
-    help.sendBackJSON(200, res, next)(null, {results})
+    help.sendBackJSON(204, res, next)(null, {results})
   }).catch(this.handleError(res, next))
 }
 
diff --git a/test/acceptance/acl/clients-api/roles-delete.js b/test/acceptance/acl/clients-api/roles-delete.js
index 4d15ba30..4d50568e 100644
--- a/test/acceptance/acl/clients-api/roles-delete.js
+++ b/test/acceptance/acl/clients-api/roles-delete.js
@@ -214,6 +214,7 @@ module.exports = () => {
         secret: 'someSecret',
         resources: {
           clients: {
+            read: true,
             update: true
           }
         },

From fa99d417a585cc4994ffcb9c5471bdac3339cbd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Tue, 17 Jul 2018 14:58:05 +0100
Subject: [PATCH 16/33] test: fix test

---
 test/acceptance/acl/clients-api/roles-delete.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/acceptance/acl/clients-api/roles-delete.js b/test/acceptance/acl/clients-api/roles-delete.js
index 4d50568e..41815cb2 100644
--- a/test/acceptance/acl/clients-api/roles-delete.js
+++ b/test/acceptance/acl/clients-api/roles-delete.js
@@ -31,6 +31,10 @@ module.exports = () => {
     })
   })
 
+  afterEach(done => {
+    help.removeACLData(done)
+  })
+
   describe('error states', () => {
     it('should return 401 if the request does not include a valid bearer token', done => {
       client
@@ -208,7 +212,7 @@ module.exports = () => {
   })
 
   describe('success states (the client has "update" access to the "clients" resource and has all the roles they are trying to assign)', () => {
-    it('should remove a role from a client', () => {
+    it('should remove a role from a client', done => {
       let testClient = {
         clientId: 'apiClient',
         secret: 'someSecret',

From ebc8e7a376c77e6dd1bd5bd91332457c32db27b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Tue, 17 Jul 2018 17:19:53 +0100
Subject: [PATCH 17/33] fix: unregister correct routes in documents controller

---
 dadi/lib/controller/documents.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dadi/lib/controller/documents.js b/dadi/lib/controller/documents.js
index 2c867faa..4a6e2224 100644
--- a/dadi/lib/controller/documents.js
+++ b/dadi/lib/controller/documents.js
@@ -252,7 +252,7 @@ Collection.prototype.stats = function (req, res, next) {
 
 Collection.prototype.unregisterRoutes = function (route) {
   this.server.app.unuse(`${route}/config`)
-  this.server.app.unuse(`${route}/:id(${this.ID_PATTERN})?/:action(count|stats)?`)
+  this.server.app.unuse(`${route}/:id(${this.ID_PATTERN})?/:action(count|search|stats)?`)
 }
 
 module.exports = function (model, server) {

From 3b7f31c109222b18348a919b8d1772b3ca696ff9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Tue, 17 Jul 2018 17:31:40 +0100
Subject: [PATCH 18/33] test: add debug code

---
 test/acceptance/acl/collections-api.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/acceptance/acl/collections-api.js b/test/acceptance/acl/collections-api.js
index 2fcde1a0..96dcc1a2 100644
--- a/test/acceptance/acl/collections-api.js
+++ b/test/acceptance/acl/collections-api.js
@@ -202,13 +202,16 @@ describe('Collections API', () => {
         fields: JSON.stringify({ field1: 1, title: 1 })
       }
 
+      console.log('---> 1')
       help.createACLClient(testClient).then(() => {
+        console.log('---> 2')
         client
         .post(config.get('auth.tokenUrl'))
         .set('content-type', 'application/json')
         .send(testClient)
         .expect(200)
         .end((err, res) => {
+          console.log('---> 3', err, res.statusCode, res.body)
           if (err) return done(err)
 
           let bearerToken = res.body.accessToken
@@ -219,6 +222,7 @@ describe('Collections API', () => {
           .set('content-type', 'application/json')
           .set('Authorization', `Bearer ${bearerToken}`)
           .end((err, res) => {
+            console.log('---> 4', err, res.statusCode, res.body)
             if (err) return done(err)
             res.statusCode.should.eql(200)
 

From ae2aa98386dbd7b1f88a458cd4bf867abb93b9bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Tue, 17 Jul 2018 17:38:13 +0100
Subject: [PATCH 19/33] test: fix test

---
 test/acceptance/acl/collections-api.js | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/test/acceptance/acl/collections-api.js b/test/acceptance/acl/collections-api.js
index 96dcc1a2..16d91427 100644
--- a/test/acceptance/acl/collections-api.js
+++ b/test/acceptance/acl/collections-api.js
@@ -202,27 +202,22 @@ describe('Collections API', () => {
         fields: JSON.stringify({ field1: 1, title: 1 })
       }
 
-      console.log('---> 1')
       help.createACLClient(testClient).then(() => {
-        console.log('---> 2')
         client
         .post(config.get('auth.tokenUrl'))
         .set('content-type', 'application/json')
         .send(testClient)
         .expect(200)
         .end((err, res) => {
-          console.log('---> 3', err, res.statusCode, res.body)
           if (err) return done(err)
 
           let bearerToken = res.body.accessToken
-          let query = require('querystring').stringify(params)
 
           client
-          .get(`/vtest/testdb/test-schema/?${query}`)
+          .get('/vtest/testdb/test-schema/?fields={"field1":1,"title":1}')
           .set('content-type', 'application/json')
           .set('Authorization', `Bearer ${bearerToken}`)
           .end((err, res) => {
-            console.log('---> 4', err, res.statusCode, res.body)
             if (err) return done(err)
             res.statusCode.should.eql(200)
 

From a78a769aebe7eea82bf652dd223631cf6e2fc13b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Tue, 17 Jul 2018 17:42:57 +0100
Subject: [PATCH 20/33] test: add debug code

---
 dadi/lib/controller/documents.js       | 3 +++
 test/acceptance/acl/collections-api.js | 6 +-----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/dadi/lib/controller/documents.js b/dadi/lib/controller/documents.js
index 4a6e2224..b47aee71 100644
--- a/dadi/lib/controller/documents.js
+++ b/dadi/lib/controller/documents.js
@@ -100,6 +100,8 @@ Collection.prototype.get = function (req, res, next) {
     queryOptions = queryOptions.queryOptions
   }
 
+  console.log('---> Getting...')
+
   return this.model.get({
     client: req.dadiApiClient,
     query,
@@ -108,6 +110,7 @@ Collection.prototype.get = function (req, res, next) {
   }).then(results => {
     return done(null, results)
   }).catch(error => {
+    console.log('---> ERROR:', error)
     return done(error)
   })
 }
diff --git a/test/acceptance/acl/collections-api.js b/test/acceptance/acl/collections-api.js
index 16d91427..460f5024 100644
--- a/test/acceptance/acl/collections-api.js
+++ b/test/acceptance/acl/collections-api.js
@@ -191,17 +191,13 @@ describe('Collections API', () => {
       })
     })
 
-    it('should return 200 with read permission and a field excluded', function (done) {
+    it.only('should return 200 with read permission and a field excluded', function (done) {
       let testClient = {
         clientId: 'apiClient',
         secret: 'someSecret',
         resources: { 'collection:testdb_test-schema': PERMISSIONS.READ_EXCLUDE_FIELDS }
       }
 
-      let params = {
-        fields: JSON.stringify({ field1: 1, title: 1 })
-      }
-
       help.createACLClient(testClient).then(() => {
         client
         .post(config.get('auth.tokenUrl'))

From da52dd335ef6068c95d657cd3ec2674fb381067b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= <mail@eduardoboucas.com>
Date: Wed, 18 Jul 2018 22:38:33 +0100
Subject: [PATCH 21/33] test: remove debug code from test

---
 dadi/lib/controller/documents.js       | 3 ---
 test/acceptance/acl/collections-api.js | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/dadi/lib/controller/documents.js b/dadi/lib/controller/documents.js
index b47aee71..4a6e2224 100644
--- a/dadi/lib/controller/documents.js
+++ b/dadi/lib/controller/documents.js
@@ -100,8 +100,6 @@ Collection.prototype.get = function (req, res, next) {
     queryOptions = queryOptions.queryOptions
   }
 
-  console.log('---> Getting...')
-
   return this.model.get({
     client: req.dadiApiClient,
     query,
@@ -110,7 +108,6 @@ Collection.prototype.get = function (req, res, next) {
   }).then(results => {
     return done(null, results)
   }).catch(error => {
-    console.log('---> ERROR:', error)
     return done(error)
   })
 }
diff --git a/test/acceptance/acl/collections-api.js b/test/acceptance/acl/collections-api.js
index 460f5024..320f62f3 100644
--- a/test/acceptance/acl/collections-api.js
+++ b/test/acceptance/acl/collections-api.js
@@ -191,7 +191,7 @@ describe('Collections API', () => {
       })
     })
 
-    it.only('should return 200 with read permission and a field excluded', function (done) {
+    it('should return 200 with read permission and a field excluded', function (done) {
       let testClient = {
         clientId: 'apiClient',
         secret: 'someSecret',

From 01c0dcfb9186ce35ea42918c67a91de9d5493989 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Sun, 22 Jul 2018 14:14:39 +0100
Subject: [PATCH 22/33] refactor: test connector refactor

---
 test/acceptance/search.js    | 27 +++++------
 test/test-connector/index.js | 86 ++++++++++++++++++------------------
 2 files changed, 55 insertions(+), 58 deletions(-)

diff --git a/test/acceptance/search.js b/test/acceptance/search.js
index a713c77e..a69c6871 100644
--- a/test/acceptance/search.js
+++ b/test/acceptance/search.js
@@ -1,20 +1,17 @@
-var _ = require('underscore')
-var should = require('should')
-var sinon = require('sinon')
-var fs = require('fs')
-var path = require('path')
-var request = require('supertest')
-var EventEmitter = require('events').EventEmitter
-var connection = require(__dirname + '/../../dadi/lib/model/connection')
-var config = require(__dirname + '/../../config')
-var help = require(__dirname + '/help')
-var app = require(__dirname + '/../../dadi/lib/')
-var model = require(__dirname + '/../../dadi/lib/model/')
+const app = require(__dirname + '/../../dadi/lib/')
+const config = require(__dirname + '/../../config')
+const fs = require('fs')
+const help = require(__dirname + '/help')
+const model = require(__dirname + '/../../dadi/lib/model/')
+const should = require('should')
+const sinon = require('sinon')
+const path = require('path')
+const request = require('supertest')
 
 // variables scoped for use throughout tests
-var bearerToken
-var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
-var lastModifiedAt = 0
+let bearerToken
+let connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
+let lastModifiedAt = 0
 
 describe('Search', function () {
   this.timeout(4000)
diff --git a/test/test-connector/index.js b/test/test-connector/index.js
index e6244a70..ae808ed4 100644
--- a/test/test-connector/index.js
+++ b/test/test-connector/index.js
@@ -520,48 +520,6 @@ DataStore.prototype.search = function ({ words, collection, options = {}, schema
 
   debug('search in %s for %o', collection, words)
 
-  function mapFn (document) {
-    return {
-      document: document.document,
-      word: document.word,
-      weight: document.weight
-    }
-  }
-
-  function reduceFn (documents) {
-    let matches = documents.reduce((groups, document) => {
-      let key = document.document
-
-      groups[key] = groups[key] || {
-        count: 0,
-        weight: 0
-      }
-
-      groups[key].count++
-      groups[key].weight = groups[key].weight + document.weight
-      return groups
-    }, {})
-
-    let output = []
-
-    Object.keys(matches).forEach(function (match) {
-      output.push({
-        _id: {
-          document: match
-        },
-        count: matches[match].count,
-        weight: matches[match].weight
-      })
-    })
-
-    output.sort(function (a, b) {
-      if (a.weight === b.weight) return 0
-      return a.weight < b.weight ? 1 : -1
-    })
-
-    return output
-  }
-
   return new Promise((resolve, reject) => {
     this.getCollection(collection).then(collection => {
       let results
@@ -574,7 +532,7 @@ DataStore.prototype.search = function ({ words, collection, options = {}, schema
 
       let baseResultset = collection.chain().find(query)
 
-      return resolve(baseResultset.mapReduce(mapFn, reduceFn))
+      return resolve(baseResultset.mapReduce(searchMapFn, searchReduceFn))
     })
   })
 }
@@ -698,6 +656,48 @@ DataStore.prototype.update = function ({query, collection, update, options = {},
   })
 }
 
+function searchMapFn (document) {
+  return {
+    document: document.document,
+    word: document.word,
+    weight: document.weight
+  }
+}
+
+function searchReduceFn (documents) {
+  let matches = documents.reduce((groups, document) => {
+    let key = document.document
+
+    groups[key] = groups[key] || {
+      count: 0,
+      weight: 0
+    }
+
+    groups[key].count++
+    groups[key].weight = groups[key].weight + document.weight
+    return groups
+  }, {})
+
+  let output = []
+
+  Object.keys(matches).forEach(function (match) {
+    output.push({
+      _id: {
+        document: match
+      },
+      count: matches[match].count,
+      weight: matches[match].weight
+    })
+  })
+
+  output.sort(function (a, b) {
+    if (a.weight === b.weight) return 0
+    return a.weight < b.weight ? 1 : -1
+  })
+
+  return output
+}
+
 module.exports = DataStore
 module.exports.settings = {
   connectWithCollection: false

From b0064f260006216f7f2b959fd43231aa014b78ec Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 25 Jul 2018 16:22:14 +0100
Subject: [PATCH 23/33] feat: include language parameter in search

---
 dadi/lib/controller/index.js                  |  25 ++-
 dadi/lib/model/search.js                      |  20 +-
 dadi/lib/search/analysers/standard.js         |   9 +-
 dadi/lib/search/index.js                      |  70 +++++--
 test/acceptance/acl/collections-api.js        | 188 +++++++++++++----
 test/acceptance/i18n.js                       |   4 +-
 test/acceptance/search.js                     | 198 +++++++++++++++++-
 .../vtest/testdb/collection.test-schema.json  |   5 +-
 8 files changed, 432 insertions(+), 87 deletions(-)

diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index b25c7696..33e3f082 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -182,13 +182,17 @@ Controller.prototype.search = function (req, res, next) {
     queryOptions = queryOptions.queryOptions
   }
 
-  return this.model.search(queryOptions).then(query => {
+  return this.model.search({
+    client: req.dadiApiClient,
+    options: queryOptions
+  }).then(query => {
     let ids = query._id['$containsAny'].map(id => id.toString())
 
-    return this.model.get({
+    return this.model.find({
+      client: req.dadiApiClient,
+      language: options.lang,
       query,
-      options: queryOptions,
-      req
+      options: queryOptions
     }).then(results => {
       results.results = results.results.sort((a, b) => {
         let aIndex = ids.indexOf(a._id.toString())
@@ -199,7 +203,18 @@ Controller.prototype.search = function (req, res, next) {
         return aIndex > bIndex ? 1 : -1
       })
 
-      return help.sendBackJSON(200, res, next)(null, results)
+      return this.model.formatForOutput(
+        results.results,
+        {
+          client: req.dadiApiClient,
+          composeOverride: queryOptions.compose,
+          language: options.lang,
+          urlFields: queryOptions.fields
+        }
+      ).then(formattedResults => {
+        results.results = formattedResults
+        return help.sendBackJSON(200, res, next)(null, results)
+      })
     })
   }).catch(error => {
     return next(error)
diff --git a/dadi/lib/model/search.js b/dadi/lib/model/search.js
index 8317649f..abe1e05e 100644
--- a/dadi/lib/model/search.js
+++ b/dadi/lib/model/search.js
@@ -9,14 +9,12 @@ const debug = require('debug')('api:model:search')
  * @param {Object} options - an options object
  * @returns {Promise<Metadata>}
  */
-module.exports = function (options = {}) {
+module.exports = function ({
+  client,
+  options = {}
+} = {}) {
   let err
 
-  if (typeof options === 'function') {
-    // done = options
-    options = {}
-  }
-
   if (!this.searchHandler.canUse()) {
     err = new Error('Not Implemented')
     err.statusCode = 501
@@ -39,7 +37,11 @@ module.exports = function (options = {}) {
     return Promise.reject(err)
   }
 
-  debug(options.search)
-
-  return this.searchHandler.find(options.search)
+  return this.validateAccess({
+    client,
+    type: 'read'
+  }).then(() => {
+    debug(options.search)
+    return this.searchHandler.find(options.search)
+  })
 }
diff --git a/dadi/lib/search/analysers/standard.js b/dadi/lib/search/analysers/standard.js
index 83cfc942..dfcd27a9 100644
--- a/dadi/lib/search/analysers/standard.js
+++ b/dadi/lib/search/analysers/standard.js
@@ -55,8 +55,11 @@ class StandardAnalyzer {
   }
 
   areValidWords (words) {
-    return Array.isArray(words) &&
-    words.every(word => {
+    if (!Array.isArray(words)) {
+      return false
+    }
+
+    return words.every(word => {
       return typeof word === 'object' &&
         word.hasOwnProperty('weight') &&
         word.hasOwnProperty('word')
@@ -93,7 +96,7 @@ class StandardAnalyzer {
 
     let docWords = this.tfidf.documents
       .map((doc, index) => {
-        let rules = this.fieldRules[doc.__key]
+        let rules = this.fieldRules[doc.__key.split(':')[0]]
 
         return words
           .filter(word => doc[word])
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index 12d64060..f5105c0d 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -17,7 +17,9 @@ const pageLimit = 20
  * N.B. May only be used with the MongoDB Data Connector.
  */
 const Search = function (model) {
-  if (!model || model.constructor.name !== 'Model') throw new Error('model should be an instance of Model')
+  if (!model || model.constructor.name !== 'Model') {
+    throw new Error('model should be an instance of Model')
+  }
 
   this.model = model
   this.indexableFields = this.getIndexableFields()
@@ -44,6 +46,8 @@ Search.prototype.canUse = function () {
 
   this.datastore = DataStore(searchConfig.datastore)
 
+  console.log(this.indexableFields)
+
   return (typeof this.datastore.search !== 'undefined') &&
     searchConfig.enabled &&
     Object.keys(this.indexableFields).length > 0
@@ -164,7 +168,9 @@ Search.prototype.delete = function (documents) {
 
   debug('deleting documents from the %s index', this.searchCollection)
 
-  let deleteQueue = documents.map(document => this.clearDocumentInstances(document._id.toString()))
+  let deleteQueue = documents.map(document => {
+    return this.clearDocumentInstances(document._id.toString())
+  })
 
   return Promise.all(deleteQueue)
 }
@@ -246,11 +252,17 @@ Search.prototype.getInstancesOfWords = function (words) {
 Search.prototype.getIndexableFields = function () {
   let schema = this.model.schema
 
-  return Object.assign({}, ...Object.keys(schema)
-    .filter(key => this.hasSearchField(schema[key]))
-    .map(key => {
-      return {[key]: schema[key].search}
-    }))
+  let indexableFields = Object.keys(schema).filter(key => {
+    return this.hasSearchField(schema[key])
+  })
+
+  let fields = {}
+
+  indexableFields.forEach(key => {
+    fields[key] = schema[key].search
+  })
+
+  return fields
 }
 
 /**
@@ -273,11 +285,27 @@ Search.prototype.hasSearchField = function (field) {
 Search.prototype.removeNonIndexableFields = function (document) {
   if (typeof document !== 'object') return {}
 
-  return Object.assign({}, ...Object.keys(document)
-    .filter(key => this.indexableFields[key])
-    .map(key => {
-      return {[key]: document[key]}
-    }))
+  // set of languages configured for API, so we can keep translation fields
+  // in the document for indexing
+  let supportedLanguages = config.get('i18n.languages')
+  let fieldSeparator = config.get('i18n.fieldCharacter')
+
+  let indexableFields = Object.keys(document).filter(key => {
+    if (key.indexOf(fieldSeparator) > 0) {
+      let keyParts = key.split(fieldSeparator)
+      return this.indexableFields[keyParts[0]] && supportedLanguages.includes(keyParts[1])
+    } else {
+      return this.indexableFields[key]
+    }
+  })
+
+  let sanitisedDocument = {}
+
+  indexableFields.forEach(key => {
+    sanitisedDocument[key] = document[key]
+  })
+
+  return sanitisedDocument
 }
 
 /**
@@ -387,12 +415,12 @@ Search.prototype.clearAndInsertWordInstances = function (words, docId) {
     settings: this.getWordSchema().settings
   }).then(results => {
     // Get all word instances from Analyser
-    this.clearDocumentInstances(docId).then(response => {
+    return this.clearDocumentInstances(docId).then(response => {
       if (response.deletedCount) {
         debug('Removed %s documents from the %s index', response.deletedCount, this.searchCollection)
       }
 
-      this.insertWordInstances(results.results, docId)
+      return this.insertWordInstances(results.results, docId)
     })
   })
   .catch(err => {
@@ -494,15 +522,15 @@ Search.prototype.runBatchIndex = function (options) {
     options: options,
     schema: this.model.schema,
     settings: this.model.settings
-  }).then(results => {
-    if (results.results && results.results.length) {
-      debug(`Indexed ${results.results.length} ${results.results.length === 1 ? 'record' : 'records'} for ${this.model.name}`)
+  }).then(({metadata, results}) => {
+    if (results && results.length) {
+      debug(`Indexed ${results.length} ${results.length === 1 ? 'record' : 'records'} for ${this.model.name}`)
 
-      if (results.results.length > 0) {
-        this.index(results.results).then(response => {
-          debug(`Indexed page ${options.page}/${results.metadata.totalPages}`)
+      if (results.length > 0) {
+        this.index(results).then(response => {
+          debug(`Indexed page ${options.page}/${metadata.totalPages}`)
 
-          if (options.page * options.limit < results.metadata.totalCount) {
+          if (options.page * options.limit < metadata.totalCount) {
             return this.batchIndex(options.page + 1, options.limit)
           }
         })
diff --git a/test/acceptance/acl/collections-api.js b/test/acceptance/acl/collections-api.js
index 320f62f3..b145280f 100644
--- a/test/acceptance/acl/collections-api.js
+++ b/test/acceptance/acl/collections-api.js
@@ -20,6 +20,14 @@ let docs
 
 describe('Collections API', () => {
   before(done => {
+    config.set('search', {
+      'enabled': true,
+      'minQueryLength': 3,
+      'wordCollection': 'words',
+      'datastore': './../../../test/test-connector',
+      'database': 'testdb'
+    })
+
     app.start(err => {
       if (err) return done(err)
 
@@ -62,10 +70,114 @@ describe('Collections API', () => {
 
   after(done => {
     help.removeACLData(() => {
+      config.set('search', {
+        'enabled': false,
+        'minQueryLength': 3,
+        'wordCollection': 'words',
+        'datastore': './../../../test/test-connector',
+        'database': 'testdb'
+      })
+
       app.stop(done)
     })
   })
 
+  describe('Search', function () {
+    it('should return 403 with no permissions', function (done) {
+      let testClient = {
+        clientId: 'apiClient',
+        secret: 'someSecret',
+        resources: { 'collection:testdb_test-schema': {} }
+      }
+
+      help.createACLClient(testClient).then(() => {
+        client
+        .post(config.get('auth.tokenUrl'))
+        .set('content-type', 'application/json')
+        .send(testClient)
+        .expect(200)
+        .end((err, res) => {
+          if (err) return done(err)
+
+          let bearerToken = res.body.accessToken
+
+          client
+          .get(`/vtest/testdb/test-schema/search?q=xyz`)
+          .set('content-type', 'application/json')
+          .set('Authorization', `Bearer ${bearerToken}`)
+          .end((err, res) => {
+            if (err) return done(err)
+            console.log(res)
+            res.statusCode.should.eql(403)
+            done()
+          })
+        })
+      })
+    })
+
+    it('should return 403 with no read permission', function (done) {
+      let testClient = {
+        clientId: 'apiClient',
+        secret: 'someSecret',
+        resources: { 'collection:testdb_test-schema': PERMISSIONS.NO_READ }
+      }
+
+      help.createACLClient(testClient).then(() => {
+        client
+        .post(config.get('auth.tokenUrl'))
+        .set('content-type', 'application/json')
+        .send(testClient)
+        .expect(200)
+        .end((err, res) => {
+          if (err) return done(err)
+
+          let bearerToken = res.body.accessToken
+
+          client
+          .get(`/vtest/testdb/test-schema/search?q=xyz`)
+          .set('content-type', 'application/json')
+          .set('Authorization', `Bearer ${bearerToken}`)
+          .end((err, res) => {
+            if (err) return done(err)
+            res.statusCode.should.eql(403)
+            done()
+          })
+        })
+      })
+    })
+
+    it('should return 200 with read permission', function (done) {
+      let testClient = {
+        clientId: 'apiClient',
+        secret: 'someSecret',
+        resources: { 'collection:testdb_test-schema': PERMISSIONS.READ }
+      }
+
+      help.createACLClient(testClient).then(() => {
+        client
+        .post(config.get('auth.tokenUrl'))
+        .set('content-type', 'application/json')
+        .send(testClient)
+        .expect(200)
+        .end((err, res) => {
+          if (err) return done(err)
+
+          let bearerToken = res.body.accessToken
+
+          client
+          .get(`/vtest/testdb/test-schema/search?q=fghj`)
+          .set('content-type', 'application/json')
+          .set('Authorization', `Bearer ${bearerToken}`)
+          .end((err, res) => {
+            if (err) return done(err)
+            res.statusCode.should.eql(200)
+            done()
+          })
+        })
+      })
+    })
+  })
+
   describe('GET', function () {
     it('should return 403 with no permissions', function (done) {
       let testClient = {
@@ -289,7 +401,7 @@ describe('Collections API', () => {
               done()
             })
           })
-        })        
+        })
       })
     })
 
@@ -357,7 +469,7 @@ describe('Collections API', () => {
               done()
             })
           })
-        })        
+        })
       })
     })
 
@@ -510,9 +622,9 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "POST",
-        "PUT",
-        "DELETE"
+        'POST',
+        'PUT',
+        'DELETE'
       ]
 
       client
@@ -532,10 +644,10 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "POST",
-        "PUT",
-        "DELETE"
+        'GET',
+        'POST',
+        'PUT',
+        'DELETE'
       ]
 
       client
@@ -699,7 +811,7 @@ describe('Collections API', () => {
         { field1: 'Value one' },
         { field1: 'Value two' },
         { field1: 'Value three' }
-      ]      
+      ]
 
       help.getBearerTokenWithPermissions({
         accessType: 'admin'
@@ -764,7 +876,7 @@ describe('Collections API', () => {
         { field1: 'Value two' },
         { field1: 'Value three' },
         { field1: 'Value four' }
-      ]      
+      ]
 
       help.getBearerTokenWithPermissions({
         accessType: 'admin'
@@ -804,9 +916,9 @@ describe('Collections API', () => {
               })
             })
           })
-        })        
+        })
       })
-    })    
+    })
   })
 
   describe('POST', function () {
@@ -938,9 +1050,9 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "PUT",
-        "DELETE"
+        'GET',
+        'PUT',
+        'DELETE'
       ]
 
       client
@@ -963,10 +1075,10 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "POST",
-        "PUT",
-        "DELETE"
+        'GET',
+        'POST',
+        'PUT',
+        'DELETE'
       ]
 
       client
@@ -985,7 +1097,7 @@ describe('Collections API', () => {
 
         done()
       })
-    })    
+    })
   })
 
   describe('PUT', function () {
@@ -1277,7 +1389,7 @@ describe('Collections API', () => {
           })
         })
       })
-    })    
+    })
 
     it('should return 200 and not update any documents when the query differs from the filter permission', function (done) {
       let testClient = {
@@ -1373,9 +1485,9 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "POST",
-        "DELETE"
+        'GET',
+        'POST',
+        'DELETE'
       ]
 
       client
@@ -1402,10 +1514,10 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "POST",
-        "PUT",
-        "DELETE"
+        'GET',
+        'POST',
+        'PUT',
+        'DELETE'
       ]
 
       client
@@ -1428,7 +1540,7 @@ describe('Collections API', () => {
 
         done()
       })
-    })    
+    })
   })
 
   describe('DELETE', function () {
@@ -1676,9 +1788,9 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "POST",
-        "PUT"
+        'GET',
+        'POST',
+        'PUT'
       ]
 
       client
@@ -1705,10 +1817,10 @@ describe('Collections API', () => {
       let modelSettings = Object.assign({}, app.components['/vtest/testdb/test-schema'].model.settings)
 
       app.components['/vtest/testdb/test-schema'].model.settings.authenticate = [
-        "GET",
-        "POST",
-        "PUT",
-        "DELETE"
+        'GET',
+        'POST',
+        'PUT',
+        'DELETE'
       ]
 
       client
@@ -1731,6 +1843,6 @@ describe('Collections API', () => {
 
         done()
       })
-    })    
+    })
   })
 })
diff --git a/test/acceptance/i18n.js b/test/acceptance/i18n.js
index 18d92105..afabe33d 100644
--- a/test/acceptance/i18n.js
+++ b/test/acceptance/i18n.js
@@ -405,7 +405,7 @@ describe('Multi-language', function () {
         done()
       })
     })
-  })  
+  })
 
   it('should populate a `_i18n` field with a mapping of the language used for each translatable field', done => {
     config.set('i18n.languages', ['pt', 'fr'])
@@ -699,7 +699,7 @@ describe('Multi-language', function () {
             if (++i === Object.keys(translations).length) {
               config.set('i18n.languages', configBackup.i18n.languages)
 
-              done()  
+              done()
             }
           })
         })
diff --git a/test/acceptance/search.js b/test/acceptance/search.js
index a69c6871..4da90ccf 100644
--- a/test/acceptance/search.js
+++ b/test/acceptance/search.js
@@ -1,17 +1,14 @@
-const app = require(__dirname + '/../../dadi/lib/')
-const config = require(__dirname + '/../../config')
-const fs = require('fs')
-const help = require(__dirname + '/help')
-const model = require(__dirname + '/../../dadi/lib/model/')
+const app = require('../../dadi/lib/')
+const config = require('../../config')
+const help = require('./help')
+const model = require('../../dadi/lib/model/')
 const should = require('should')
-const sinon = require('sinon')
-const path = require('path')
 const request = require('supertest')
 
 // variables scoped for use throughout tests
 let bearerToken
 let connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port')
-let lastModifiedAt = 0
+let configBackup = config.get()
 
 describe('Search', function () {
   this.timeout(4000)
@@ -30,6 +27,8 @@ describe('Search', function () {
         'database': 'testdb'
       })
 
+      config.set('i18n.languages', ['fr', 'pt'])
+
       app.start(function () {
         help.getBearerTokenWithAccessType('admin', function (err, token) {
           if (err) return done(err)
@@ -95,6 +94,8 @@ describe('Search', function () {
       'enabled': false
     })
 
+    config.set('i18n.languages', configBackup.i18n.languages)
+
     app.stop(() => {
       cleanupFn()
       done()
@@ -230,4 +231,185 @@ describe('Search', function () {
       })
     })
   })
+
+  describe('Multi-language', function () {
+    it('should retrieve all language variations if no `lang` parameter is supplied', done => {
+      let document = {
+        title: 'The Little Prince',
+        'title:pt': 'O Principezinho',
+        'title:fr': 'Le Petit Prince'
+      }
+
+      var client = request(connectionString)
+
+      client
+      .post('/vtest/testdb/test-schema')
+      .set('Authorization', `Bearer ${bearerToken}`)
+      .send(document)
+      .expect(200)
+      .end((err, res) => {
+        if (err) return done(err)
+
+        client
+        .get(`/vtest/testdb/test-schema/search?q=Prince`)
+        .set('Authorization', `Bearer ${bearerToken}`)
+        .expect(200)
+        .end((err, res) => {
+          res.body.results.length.should.eql(1)
+
+          let result = res.body.results[0]
+
+          result.title.should.eql(document.title)
+          result['title:pt'].should.eql(document['title:pt'])
+          result['title:fr'].should.eql(document['title:fr'])
+
+          should.not.exist(result._i18n)
+
+          done()
+        })
+      })
+    })
+
+    it('should return the translation version of a field when there is one set for the language in the `lang` parameter, falling back to the default language', done => {
+      config.set('i18n.languages', ['pt', 'fr'])
+
+      let documents = [
+        {
+          title: 'The Little Prince',
+          'title:pt': 'O Principezinho',
+          'title:fr': 'Le Petit Prince'
+        },
+        {
+          title: 'The Untranslatable'
+        }
+      ]
+
+      var client = request(connectionString)
+
+      client
+      .post(`/vtest/testdb/test-schema`)
+      .set('Authorization', `Bearer ${bearerToken}`)
+      .send(documents)
+      .expect(200)
+      .end((err, res) => {
+        if (err) return done(err)
+
+        client
+        .get('/vtest/testdb/test-schema/search?q=Principezinho&lang=pt')
+        .set('Authorization', `Bearer ${bearerToken}`)
+        .expect(200)
+        .end((err, res) => {
+          res.body.results.length.should.eql(2)
+
+          let results = res.body.results
+
+          results[0].title.should.eql(documents[0]['title:pt'])
+          results[0]._i18n.title.should.eql('pt')
+          should.not.exist(results[0]['title:pt'])
+          should.not.exist(results[0]['title:fr'])
+
+          // results[1].title.should.eql(documents[1].title)
+          // results[1]._i18n.title.should.eql(
+          //   config.get('i18n.defaultLanguage')
+          // )
+          // should.not.exist(results[1]['title:pt'])
+          // should.not.exist(results[1]['title:fr'])
+
+          config.set('i18n.languages', configBackup.i18n.languages)
+
+          done()
+        })
+      })
+    })
+
+    it('should return the translation version of a field when the fields projection is set to include the field in question', done => {
+      config.set('i18n.languages', ['pt', 'fr'])
+
+      let documents = [
+        {
+          title: 'The Little Prince',
+          'title:pt': 'O Principezinho',
+          'title:fr': 'Le Petit Prince'
+        },
+        {
+          title: 'The Untranslatable'
+        }
+      ]
+
+      var client = request(connectionString)
+
+      client
+      .post('/vtest/testdb/test-schema')
+      .set('Authorization', `Bearer ${bearerToken}`)
+      .send(documents)
+      .expect(200)
+      .end((err, res) => {
+        if (err) return done(err)
+
+        client
+        .get(`/vtest/testdb/test-schema/search?q=Principezinho&fields={"title":1}&lang=pt`)
+        .set('Authorization', `Bearer ${bearerToken}`)
+        .expect(200)
+        .end((err, res) => {
+          let results = res.body.results
+
+          results[0].title.should.eql(documents[0]['title:pt'])
+          results[0]._i18n.title.should.eql('pt')
+          should.not.exist(results[0]['title:pt'])
+          should.not.exist(results[0]['title:fr'])
+
+          // results[1].title.should.eql(documents[1].title)
+          // results[1]._i18n.title.should.eql(
+          //   config.get('i18n.defaultLanguage')
+          // )
+          // should.not.exist(results[1]['title:pt'])
+          // should.not.exist(results[1]['title:fr'])
+
+          config.set('i18n.languages', configBackup.i18n.languages)
+
+          done()
+        })
+      })
+    })
+
+    it('should return the original version of a field when the requested language is not part of `i18n.languages`', done => {
+      config.set('i18n.languages', ['fr'])
+
+      let document = {
+        title: 'The Little Prince',
+        'title:pt': 'O Principezinho',
+        'title:fr': 'Le Petit Prince'
+      }
+
+      var client = request(connectionString)
+
+      client
+      .post('/vtest/testdb/test-schema')
+      .set('Authorization', `Bearer ${bearerToken}`)
+      .send(document)
+      .expect(200)
+      .end((err, res) => {
+        if (err) return done(err)
+
+        client
+        .get(`/vtest/testdb/test-schema/search?q=Prince&fields={"title":1}&lang=pt`)
+        .set('Authorization', `Bearer ${bearerToken}`)
+        .expect(200)
+        .end((err, res) => {
+          // res.body.results.length.should.eql(1)
+
+          let results = res.body.results
+
+          results[0].title.should.eql(document.title)
+          results[0]._i18n.title.should.eql('en')
+          should.not.exist(results[0]['title:pt'])
+          should.not.exist(results[0]['title:fr'])
+
+          config.set('i18n.languages', configBackup.i18n.languages)
+
+          done()
+        })
+      })
+    })
+  })
 })
diff --git a/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json b/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
index c59f8aba..e3dcf1e5 100755
--- a/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
+++ b/test/acceptance/workspace/collections/vtest/testdb/collection.test-schema.json
@@ -12,7 +12,10 @@
       "label": "Title",
       "comments": "The title of the entry",
       "validation": {},
-      "required": false
+      "required": false,
+      "search": {
+        "weight": 2
+      }
     }
   },
   "settings": {

From 4786c10e025affed78e827336c88235b1c0d0bd5 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 25 Jul 2018 16:42:55 +0100
Subject: [PATCH 24/33] refactor: remove promise catch

---
 dadi/lib/search/index.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index f5105c0d..199791b8 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -46,8 +46,6 @@ Search.prototype.canUse = function () {
 
   this.datastore = DataStore(searchConfig.datastore)
 
-  console.log(this.indexableFields)
-
   return (typeof this.datastore.search !== 'undefined') &&
     searchConfig.enabled &&
     Object.keys(this.indexableFields).length > 0
@@ -146,8 +144,6 @@ Search.prototype.find = function (searchTerm) {
           '$containsAny': wordInstances
         }
       }
-    }).catch(err => {
-      console.log(err)
     })
   })
 }
@@ -163,7 +159,7 @@ Search.prototype.delete = function (documents) {
   }
 
   if (!Array.isArray(documents)) {
-    return
+    return Promise.resolve()
   }
 
   debug('deleting documents from the %s index', this.searchCollection)

From 90bc3be8ec5afcd06f982c1eb20f63a0f11b369e Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Sat, 28 Jul 2018 22:33:54 +0100
Subject: [PATCH 25/33] fix: return acl error correctly

---
 dadi/lib/controller/index.js           |  2 +-
 test/acceptance/acl/collections-api.js |  1 -
 test/unit/model/index.js               | 30 --------------------------
 test/unit/search/index.js              | 25 ---------------------
 4 files changed, 1 insertion(+), 57 deletions(-)

diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js
index 33e3f082..e2c45546 100755
--- a/dadi/lib/controller/index.js
+++ b/dadi/lib/controller/index.js
@@ -217,7 +217,7 @@ Controller.prototype.search = function (req, res, next) {
       })
     })
   }).catch(error => {
-    return next(error)
+    return help.sendBackJSON(null, res, next)(error)
   })
 }
 
diff --git a/test/acceptance/acl/collections-api.js b/test/acceptance/acl/collections-api.js
index b145280f..ac68a381 100644
--- a/test/acceptance/acl/collections-api.js
+++ b/test/acceptance/acl/collections-api.js
@@ -107,7 +107,6 @@ describe('Collections API', () => {
           .set('Authorization', `Bearer ${bearerToken}`)
           .end((err, res) => {
             if (err) return done(err)
-            console.log(res)
             res.statusCode.should.eql(403)
             done()
           })
diff --git a/test/unit/model/index.js b/test/unit/model/index.js
index 7e961601..99b94289 100755
--- a/test/unit/model/index.js
+++ b/test/unit/model/index.js
@@ -70,36 +70,6 @@ describe('Model', function () {
       done()
     })
 
-    it.skip('should accept database connection as third argument', function (done) {
-      config.set('database.enableCollectionDatabases', true)
-      connection.resetConnections()
-
-      const conn = connection({
-        'username': '',
-        'password': '',
-        'database': 'test',
-        'replicaSet': '',
-        'hosts': [
-          {
-            'host': 'localhost',
-            'port': 27020
-          }
-        ]
-      })
-
-      // TODO: stub the connect method so this doesn't cause a connection attempt
-
-      const mod = model('testModelName', help.getModelSchema(), conn)
-      should.exist(mod.connection)
-      mod.connection.connectionOptions.hosts[0].host.should.equal('localhost')
-      mod.connection.connectionOptions.hosts[0].port.should.equal(27020)
-      mod.connection.connectionOptions.database.should.equal('test')
-
-      config.set('database.enableCollectionDatabases', false)
-
-      done()
-    })
-
     it('should accept model settings as fourth argument', function (done) {
       const mod = model(
         'testModelName',
diff --git a/test/unit/search/index.js b/test/unit/search/index.js
index 8903d201..8320a8f2 100644
--- a/test/unit/search/index.js
+++ b/test/unit/search/index.js
@@ -144,19 +144,6 @@ describe('Search', () => {
     })
   })
 
-  describe.skip('`runFind` method', () => {
-    it('should search the database based on the query', done => {
-      const dbFindStub = sinon.spy(store.prototype, 'find')
-
-      searchInstance.runFind(searchInstance.model.connection.db, {foo: 'bar'}, searchInstance.model.name, searchInstance.model.schema, {})
-      dbFindStub.called.should.be.true
-      dbFindStub.lastCall.args[0].should.have.property('query', {foo: 'bar'})
-      dbFindStub.restore()
-
-      done()
-    })
-  })
-
   describe('`clearDocumentInstances` method', () => {
     it('should delete all search instance documents with filtered query', done => {
       const dbDeleteStub = sinon.spy(store.prototype, 'delete')
@@ -193,18 +180,6 @@ describe('Search', () => {
     })
   })
 
-  describe.skip('`insert` method', () => {
-    it('should not execute the database insert if no data is provided', done => {
-      const dbInsertStub = sinon.spy(store.prototype, 'insert')
-
-      searchInstance.insert({}, {}, {}, {}, {})
-      dbInsertStub.called.should.be.false
-      dbInsertStub.restore()
-
-      done()
-    })
-  })
-
   describe('`batchIndex` method', () => {
     it('should not execute the runBatchIndex method if no fields can be indexed', done => {
       let schema = help.getSearchModelSchema()

From af244b627c13c37cf1f5718383b177d34578d3d3 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Mon, 30 Jul 2018 11:15:17 +0100
Subject: [PATCH 26/33] fix: use regex tokenizer for accented characters

---
 dadi/lib/search/analysers/standard.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/dadi/lib/search/analysers/standard.js b/dadi/lib/search/analysers/standard.js
index dfcd27a9..d64758f0 100644
--- a/dadi/lib/search/analysers/standard.js
+++ b/dadi/lib/search/analysers/standard.js
@@ -2,12 +2,16 @@
 
 const natural = require('natural')
 const TfIdf = natural.TfIdf
-const tokenizer = new natural.WordTokenizer()
+const tokenizer = new natural.RegexpTokenizer({
+  // pattern: new RegExp(/[^A-Za-zÅåÀÈÌÒÙàèìòùÁÉÍÓÚÝáéíóúýÂÊÎÔÛâêîôûÃÑÕãñõÄËÏÖÜŸäëïöüÿŠŽšžÇç]/i)
+  pattern: new RegExp(/[^a-zA-Z\u00C0-\u017F]/i)
+})
 
 class StandardAnalyzer {
   constructor (fieldRules) {
     this.fieldRules = fieldRules
     this.tfidf = new TfIdf()
+    this.tfidf.setTokenizer(tokenizer)
   }
 
   add (field, value) {

From 906146e096893b67a2c4ce51040f74bde51016ca Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Mon, 30 Jul 2018 14:39:07 +0100
Subject: [PATCH 27/33] fix: analyse words for current document only

---
 dadi/lib/search/index.js | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index 199791b8..e7332281 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -368,11 +368,21 @@ Search.prototype.indexDocument = function (document) {
  * @return {Array} A list of analysed words.
  */
 Search.prototype.analyseDocumentWords = function (doc) {
+  // add the document to the analyser index
   Object.keys(doc).map(key => {
     this.analyser.add(key, doc[key])
   })
 
-  return this.analyser.getAllWords()
+  // add the document to a fresh analyser instance so we can get only the
+  // indexable words from THIS DOCUMENT
+  let analyser = new DefaultAnalyser(this.indexableFields)
+
+  Object.keys(doc).map(key => {
+    analyser.add(key, doc[key])
+  })
+
+  // return indexable words from THIS DOCUMENT only
+  return analyser.getAllWords()
 }
 
 /**
@@ -425,7 +435,6 @@ Search.prototype.clearAndInsertWordInstances = function (words, docId) {
 }
 
 /**
- * Insert Word Instance
  * Insert Document word instances.
  * @param  {Class} analyser Instance of document populated analyser class.
  * @param  {[type]} words Results from database query for word list.

From 49c51a95ce24bf0dd5627d05ba8811f318e401cf Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 31 Jul 2018 20:01:36 +0100
Subject: [PATCH 28/33] feat: add indexing endpoint

---
 dadi/lib/controller/searchIndex.js | 44 ++++++++++++++++++++++++++++++
 dadi/lib/index.js                  |  2 ++
 dadi/lib/search/index.js           |  1 +
 3 files changed, 47 insertions(+)
 create mode 100644 dadi/lib/controller/searchIndex.js

diff --git a/dadi/lib/controller/searchIndex.js b/dadi/lib/controller/searchIndex.js
new file mode 100644
index 00000000..363669da
--- /dev/null
+++ b/dadi/lib/controller/searchIndex.js
@@ -0,0 +1,44 @@
+const acl = require('./../model/acl')
+const config = require('./../../../config')
+const help = require('./../help')
+
+const SearchIndex = function (server) {
+  this.server = server
+
+  server.app.routeMethods('/api/index', {
+    post: this.post.bind(this)
+  })
+}
+
+SearchIndex.prototype.post = function (req, res, next) {
+  if (!req.dadiApiClient.clientId) {
+    return help.sendBackJSON(null, res, next)(
+      acl.createError(req.dadiApiClient)
+    )
+  }
+
+  // 404 if Search is not enabled
+  if (config.get('search.enabled') === false) {
+    return next()
+  }
+
+  res.statusCode = 204
+  res.end(JSON.stringify({'message': 'Indexing started'}))
+
+  try {
+    Object.keys(this.server.components).forEach(key => {
+      let value = this.server.components[key]
+
+      let hasModel = Object.keys(value).includes('model') &&
+        value.model.constructor.name === 'Model'
+
+      if (hasModel) {
+        value.model.searchHandler.batchIndex()
+      }
+    })
+  } catch (err) {
+    console.log(err)
+  }
+}
+
+module.exports = server => new SearchIndex(server)
diff --git a/dadi/lib/index.js b/dadi/lib/index.js
index 3071ba68..48380779 100755
--- a/dadi/lib/index.js
+++ b/dadi/lib/index.js
@@ -30,6 +30,7 @@ var LanguagesController = require(path.join(__dirname, '/controller/languages'))
 var MediaController = require(path.join(__dirname, '/controller/media'))
 var ResourcesController = require(path.join(__dirname, '/controller/resources'))
 var RolesController = require(path.join(__dirname, '/controller/roles'))
+var SearchIndexController = require(path.join(__dirname, '/controller/searchIndex'))
 var StatusEndpointController = require(path.join(__dirname, '/controller/status'))
 var dadiBoot = require('@dadi/boot')
 var help = require(path.join(__dirname, '/help'))
@@ -262,6 +263,7 @@ Server.prototype.start = function (done) {
   LanguagesController(this)
   ResourcesController(this)
   RolesController(this)
+  SearchIndexController(this)
 
   this.readyState = 1
 
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index e7332281..f19aae59 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -505,6 +505,7 @@ Search.prototype.batchIndex = function (page = 1, limit = 1000) {
   }
 
   debug(`Indexing page ${page} (${limit} per page)`)
+  console.log(`Indexing page ${page} (${limit} per page)`)
 
   if (this.model.connection.db) {
     this.runBatchIndex(options)

From d2295834f40055d712e679fc27abe1cf7bfaaf2f Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Tue, 31 Jul 2018 20:03:00 +0100
Subject: [PATCH 29/33] chore: package lock

---
 package-lock.json | 389 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 356 insertions(+), 33 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index e74c2df2..4d4f8647 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,13 +1,13 @@
 {
   "name": "@dadi/api",
-  "version": "4.0.0-rc3",
+  "version": "4.0.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
     "@commitlint/cli": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-4.1.1.tgz",
-      "integrity": "sha512-kt4Ib/h6yRGr+vqc+uV8uHzq4s9tbOQffookE+SphDS9FvtZt1UUgBdNlOe4V4bkY6qKocR+RlNRw1+gOCw3hg==",
+      "integrity": "sha1-NJAonpCBegrCtrkRurFLrdjegPU=",
       "dev": true,
       "requires": {
         "@commitlint/core": "4.3.0",
@@ -41,7 +41,7 @@
     "@commitlint/core": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/@commitlint/core/-/core-4.3.0.tgz",
-      "integrity": "sha512-oWAlGWIOoquQVErLeAXFUOlAQDVJxa0196z7kt/BNcEGjfqRrEwxncZ9GFjycGYb0CyS/YQ1VDe4l8YfrSmbQg==",
+      "integrity": "sha1-6IGgoWWUrzreCb5NErdwuZE7wmE=",
       "dev": true,
       "requires": {
         "@marionebl/conventional-commits-parser": "3.0.0",
@@ -161,6 +161,12 @@
         "moment": "2.19.3"
       }
     },
+    "@dadi/metadata": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@dadi/metadata/-/metadata-2.0.0.tgz",
+      "integrity": "sha512-GI0v4QEROhkDeIKmfMrHD8+9rOdgc8KbLLIZcY/FU5pYfudil7Njj+8DW2vId1tUrUvbZn3875h8TuEN9Zkrig==",
+      "dev": true
+    },
     "@dadi/status": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/@dadi/status/-/status-1.0.3.tgz",
@@ -206,7 +212,7 @@
     "@marionebl/git-raw-commits": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@marionebl/git-raw-commits/-/git-raw-commits-1.2.0.tgz",
-      "integrity": "sha512-kI7s1W+GFMLJkuesgBdMgr1NCkChqfhP+wT6REoPsgtJGGwN0L/84gSw9pyH3u1bAK3uHjAkGZQ2bileBVVWtg==",
+      "integrity": "sha1-fNim38Calt+Y2PvpF1xZccwHyCs=",
       "dev": true,
       "requires": {
         "dargs": "4.1.0",
@@ -230,7 +236,7 @@
         "glob": {
           "version": "7.1.2",
           "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
-          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "integrity": "sha1-wZyd+aAocC1nhhI4SmVSQExjbRU=",
           "dev": true,
           "requires": {
             "fs.realpath": "1.0.0",
@@ -244,7 +250,7 @@
         "minimatch": {
           "version": "3.0.4",
           "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-          "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+          "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=",
           "dev": true,
           "requires": {
             "brace-expansion": "1.1.11"
@@ -253,7 +259,7 @@
         "rimraf": {
           "version": "2.6.2",
           "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz",
-          "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==",
+          "integrity": "sha1-LtgVDSShbqhlHm1u8PR8QVjOejY=",
           "dev": true,
           "requires": {
             "glob": "7.1.2"
@@ -300,6 +306,11 @@
         }
       }
     },
+    "afinn-165": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.2.tgz",
+      "integrity": "sha512-oVbXkteWA6XgYndv3dXYVvulStflVYQtR2K+zp2PyaVhPkkOhZ8tAvk9V7cwaI43GwZaNqRoC2VTpoaWmFyBTA=="
+    },
     "ajv": {
       "version": "5.5.2",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
@@ -373,6 +384,14 @@
         "normalize-path": "2.1.1"
       }
     },
+    "apparatus": {
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz",
+      "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==",
+      "requires": {
+        "sylvester": "0.0.12"
+      }
+    },
     "append-transform": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-0.4.0.tgz",
@@ -958,6 +977,12 @@
         }
       }
     },
+    "browser-stdout": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz",
+      "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==",
+      "dev": true
+    },
     "buffer": {
       "version": "4.9.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
@@ -1263,7 +1288,7 @@
     "color-convert": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz",
-      "integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==",
+      "integrity": "sha1-wSYRB66y8pTr/+ye2eytUppgl+0=",
       "requires": {
         "color-name": "1.1.3"
       }
@@ -1302,6 +1327,12 @@
         "dot-prop": "3.0.0"
       }
     },
+    "component-emitter": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
+      "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=",
+      "dev": true
+    },
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -1360,7 +1391,7 @@
     "conventional-changelog-angular": {
       "version": "1.6.2",
       "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-1.6.2.tgz",
-      "integrity": "sha512-LiGZkMJOCJFLNzDlZo3f+DpblcDSzsaYHUWhC+kzsqq+no4qwDP3uW0HVIHueXT4jJDhYNaE9t/XCD7vu7xR1g==",
+      "integrity": "sha1-CoETE95GMm5eThHawoHWHP4fAMQ=",
       "dev": true,
       "requires": {
         "compare-func": "1.3.2",
@@ -1393,6 +1424,12 @@
         }
       }
     },
+    "cookiejar": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz",
+      "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==",
+      "dev": true
+    },
     "copy-descriptor": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
@@ -1412,7 +1449,7 @@
     "cosmiconfig": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-3.1.0.tgz",
-      "integrity": "sha512-zedsBhLSbPBms+kE7AH4vHg6JsKDz6epSv2/+5XHs8ILHlgDciSJfSWf8sX9aQ52Jb7KI7VswUTsLpR/G0cr2Q==",
+      "integrity": "sha1-ZAqUv5hH8yGABAPNJzr2BmXHM5c=",
       "dev": true,
       "requires": {
         "is-directory": "0.3.1",
@@ -1432,6 +1469,88 @@
         }
       }
     },
+    "coveralls": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-3.0.2.tgz",
+      "integrity": "sha512-Tv0LKe/MkBOilH2v7WBiTBdudg2ChfGbdXafc/s330djpF3zKOmuehTeRwjXWc7pzfj9FrDUTA7tEx6Div8NFw==",
+      "dev": true,
+      "requires": {
+        "growl": "1.10.5",
+        "js-yaml": "3.12.0",
+        "lcov-parse": "0.0.10",
+        "log-driver": "1.2.7",
+        "minimist": "1.2.0",
+        "request": "2.87.0"
+      },
+      "dependencies": {
+        "esprima": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+          "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+          "dev": true
+        },
+        "form-data": {
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz",
+          "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=",
+          "dev": true,
+          "requires": {
+            "asynckit": "0.4.0",
+            "combined-stream": "1.0.6",
+            "mime-types": "2.1.17"
+          },
+          "dependencies": {
+            "combined-stream": {
+              "version": "1.0.6",
+              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz",
+              "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=",
+              "dev": true,
+              "requires": {
+                "delayed-stream": "1.0.0"
+              }
+            }
+          }
+        },
+        "js-yaml": {
+          "version": "3.12.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz",
+          "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==",
+          "dev": true,
+          "requires": {
+            "argparse": "1.0.9",
+            "esprima": "4.0.1"
+          }
+        },
+        "request": {
+          "version": "2.87.0",
+          "resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz",
+          "integrity": "sha512-fcogkm7Az5bsS6Sl0sibkbhcKsnyon/jV1kF3ajGmF0c8HrttdKTPRT9hieOaQHA5HEq6r8OyWOo/o781C1tNw==",
+          "dev": true,
+          "requires": {
+            "aws-sign2": "0.7.0",
+            "aws4": "1.6.0",
+            "caseless": "0.12.0",
+            "combined-stream": "1.0.5",
+            "extend": "3.0.1",
+            "forever-agent": "0.6.1",
+            "form-data": "2.3.2",
+            "har-validator": "5.0.3",
+            "http-signature": "1.2.0",
+            "is-typedarray": "1.0.0",
+            "isstream": "0.1.2",
+            "json-stringify-safe": "5.0.1",
+            "mime-types": "2.1.17",
+            "oauth-sign": "0.8.2",
+            "performance-now": "2.1.0",
+            "qs": "6.5.1",
+            "safe-buffer": "5.1.1",
+            "tough-cookie": "2.3.3",
+            "tunnel-agent": "0.6.0",
+            "uuid": "3.3.2"
+          }
+        }
+      }
+    },
     "create-error-class": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -1443,7 +1562,7 @@
     "create-react-class": {
       "version": "15.6.3",
       "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz",
-      "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==",
+      "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=",
       "dev": true,
       "requires": {
         "fbjs": "0.8.16",
@@ -1535,7 +1654,7 @@
     "debug": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-      "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+      "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=",
       "requires": {
         "ms": "2.0.0"
       }
@@ -1738,6 +1857,12 @@
         }
       }
     },
+    "diff": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
+      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
+      "dev": true
+    },
     "doctrine": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz",
@@ -1985,7 +2110,7 @@
         "debug": {
           "version": "2.6.9",
           "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=",
           "dev": true,
           "requires": {
             "ms": "2.0.0"
@@ -2287,6 +2412,12 @@
       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
       "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
     },
+    "faker": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/faker/-/faker-4.1.0.tgz",
+      "integrity": "sha1-HkW7vsxndLPBlfrSg1EJxtdIzD8=",
+      "dev": true
+    },
     "fakeredis": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/fakeredis/-/fakeredis-1.0.3.tgz",
@@ -2509,6 +2640,12 @@
         "samsam": "1.1.2"
       }
     },
+    "formidable": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz",
+      "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg==",
+      "dev": true
+    },
     "fragment-cache": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
@@ -3009,7 +3146,7 @@
     "fsu": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/fsu/-/fsu-1.0.4.tgz",
-      "integrity": "sha512-T8DGjqL3DNJsA/uHWUTIZhJ/VuEqi3QdNsQBAWpKtoIPS/8rK4HWG79ae2+HEw+Cz9e5lIsWghpoXCcNsrDPFA==",
+      "integrity": "sha1-WGqPvY0ZrN8zDOy88X1kHpw3C6A=",
       "dev": true
     },
     "generate-function": {
@@ -3167,6 +3304,12 @@
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz",
       "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg="
     },
+    "growl": {
+      "version": "1.10.5",
+      "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
+      "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==",
+      "dev": true
+    },
     "handlebars": {
       "version": "4.0.11",
       "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.11.tgz",
@@ -3263,6 +3406,12 @@
         "sntp": "2.1.0"
       }
     },
+    "he": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz",
+      "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=",
+      "dev": true
+    },
     "hoek": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz",
@@ -4009,6 +4158,11 @@
       "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
       "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
     },
+    "js-promise-queue": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/js-promise-queue/-/js-promise-queue-1.1.0.tgz",
+      "integrity": "sha512-cPGuny7ogxJ8StsPKMxcWC4d6g5xngiYgbtVXitksYIsKM05KBKLzZLxMr7rWTulCp0wEcJ8Nh973z4Do1I2lg=="
+    },
     "js-tokens": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
@@ -4059,7 +4213,6 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
       "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=",
-      "dev": true,
       "requires": {
         "jsonify": "0.0.0"
       }
@@ -4085,8 +4238,7 @@
     "jsonify": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
-      "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=",
-      "dev": true
+      "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM="
     },
     "jsonparse": {
       "version": "1.3.1",
@@ -4203,6 +4355,11 @@
         }
       }
     },
+    "langs": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/langs/-/langs-2.0.0.tgz",
+      "integrity": "sha1-AMMs5IFSpJphRFC5uiYyq1igo2Q="
+    },
     "latest-version": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-2.0.0.tgz",
@@ -4227,6 +4384,12 @@
         "invert-kv": "1.0.0"
       }
     },
+    "lcov-parse": {
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-0.0.10.tgz",
+      "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=",
+      "dev": true
+    },
     "length-stream": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz",
@@ -4309,7 +4472,7 @@
     "lodash.isfunction": {
       "version": "3.0.9",
       "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz",
-      "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==",
+      "integrity": "sha1-Bt4l302zJ6yTGYHRvbBn5a9o0FE=",
       "dev": true
     },
     "lodash.isinteger": {
@@ -4356,6 +4519,12 @@
         "lodash._reinterpolate": "3.0.0"
       }
     },
+    "log-driver": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz",
+      "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==",
+      "dev": true
+    },
     "log-symbols": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
@@ -4364,6 +4533,12 @@
         "chalk": "2.3.1"
       }
     },
+    "lokijs": {
+      "version": "1.5.5",
+      "resolved": "https://registry.npmjs.org/lokijs/-/lokijs-1.5.5.tgz",
+      "integrity": "sha1-HCH4KvdXkDf63nueSBNIXCNwi7Y=",
+      "dev": true
+    },
     "lolex": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.3.2.tgz",
@@ -4460,6 +4635,12 @@
       "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=",
       "dev": true
     },
+    "methods": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+      "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=",
+      "dev": true
+    },
     "micromatch": {
       "version": "3.1.10",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
@@ -4555,10 +4736,69 @@
         }
       }
     },
+    "mocha": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz",
+      "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==",
+      "dev": true,
+      "requires": {
+        "browser-stdout": "1.3.1",
+        "commander": "2.15.1",
+        "debug": "3.1.0",
+        "diff": "3.5.0",
+        "escape-string-regexp": "1.0.5",
+        "glob": "7.1.2",
+        "growl": "1.10.5",
+        "he": "1.1.1",
+        "minimatch": "3.0.4",
+        "mkdirp": "0.5.1",
+        "supports-color": "5.4.0"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "2.15.1",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
+          "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==",
+          "dev": true
+        },
+        "glob": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "dev": true,
+          "requires": {
+            "fs.realpath": "1.0.0",
+            "inflight": "1.0.6",
+            "inherits": "2.0.3",
+            "minimatch": "3.0.4",
+            "once": "1.4.0",
+            "path-is-absolute": "1.0.1"
+          }
+        },
+        "minimatch": {
+          "version": "3.0.4",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+          "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+          "dev": true,
+          "requires": {
+            "brace-expansion": "1.1.11"
+          }
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "3.0.0"
+          }
+        }
+      }
+    },
     "mochawesome": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/mochawesome/-/mochawesome-2.3.1.tgz",
-      "integrity": "sha512-amkBeQZz/IUTm2o1VLHiih30RHdt4uMAAhyvd5oJ5FMq5gCmqzFS29pobVyGajqzPvC+na8U+nzO8DtftQoLLw==",
+      "integrity": "sha1-Q7JEXkuiX1hbzaaeVZLAgV+cNwk=",
       "dev": true,
       "requires": {
         "babel-runtime": "6.26.0",
@@ -4604,7 +4844,7 @@
         "diff": {
           "version": "3.4.0",
           "resolved": "https://registry.npmjs.org/diff/-/diff-3.4.0.tgz",
-          "integrity": "sha512-QpVuMTEoJMF7cKzi6bvWhRulU1fZqZnvyVQgNhPaxxuTYwyjn/j1v9falseQ/uXWwPnO56RBfwtg4h/EQXmucA==",
+          "integrity": "sha1-sdhVB9rzlkgo3lSzfQ1zumfdpWw=",
           "dev": true
         },
         "lodash": {
@@ -4641,7 +4881,7 @@
     "mochawesome-report-generator": {
       "version": "2.3.2",
       "resolved": "https://registry.npmjs.org/mochawesome-report-generator/-/mochawesome-report-generator-2.3.2.tgz",
-      "integrity": "sha512-T2bY3ezsZuKmM9DyZff/F7WlhHVfEq2Y2dZb9PdfT+ZclFz/b7iIqXBThi6j5Y+xUSOV1LY4rg072Fc0xdmiRQ==",
+      "integrity": "sha1-OmiFmW0yg2Ej/kAttDdOkJ5UoSc=",
       "dev": true,
       "requires": {
         "chalk": "1.1.3",
@@ -4745,6 +4985,16 @@
         }
       }
     },
+    "mock-require": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/mock-require/-/mock-require-3.0.2.tgz",
+      "integrity": "sha512-aD/Y1ZFHqw5pHg3HVQ50dLbfaAAcytS6sqLuhP51Dk3TSPdFb2VkSAa3mjrHifLIlGAtwQHJHINafAyqAne7vA==",
+      "dev": true,
+      "requires": {
+        "get-caller-file": "1.0.2",
+        "normalize-path": "2.1.1"
+      }
+    },
     "module-not-found-error": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz",
@@ -4808,6 +5058,18 @@
       "integrity": "sha1-IKMYwwy0X3H+et+/eyHJnBRy7xE=",
       "dev": true
     },
+    "natural": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/natural/-/natural-0.6.1.tgz",
+      "integrity": "sha1-Iwe/BPFyAShq6hRjCgj31/vC0GA=",
+      "requires": {
+        "afinn-165": "1.0.2",
+        "apparatus": "0.0.10",
+        "json-stable-stringify": "1.0.1",
+        "sylvester": "0.0.12",
+        "underscore": "1.8.3"
+      }
+    },
     "natural-compare": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -4823,7 +5085,7 @@
     "node-fetch": {
       "version": "1.7.3",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
-      "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
+      "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=",
       "dev": true,
       "requires": {
         "encoding": "0.1.12",
@@ -5050,7 +5312,7 @@
     "p-limit": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.2.0.tgz",
-      "integrity": "sha512-Y/OtIaXtUPr4/YpMv1pCL5L5ed0rumAaAeBSj12F+bSlMdys7i8oQF/GUJmfpTS/QoaRrS/k6pma29haJpsMng==",
+      "integrity": "sha1-DpK2vty1nwIsE9DxlJ3ILRWQnxw=",
       "dev": true,
       "requires": {
         "p-try": "1.0.0"
@@ -5257,7 +5519,7 @@
     "promise": {
       "version": "7.3.1",
       "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
-      "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
+      "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=",
       "dev": true,
       "requires": {
         "asap": "2.0.6"
@@ -5838,7 +6100,7 @@
         "debug": {
           "version": "2.6.9",
           "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=",
           "requires": {
             "ms": "2.0.0"
           }
@@ -5979,7 +6241,7 @@
         "diff": {
           "version": "3.4.0",
           "resolved": "https://registry.npmjs.org/diff/-/diff-3.4.0.tgz",
-          "integrity": "sha512-QpVuMTEoJMF7cKzi6bvWhRulU1fZqZnvyVQgNhPaxxuTYwyjn/j1v9falseQ/uXWwPnO56RBfwtg4h/EQXmucA==",
+          "integrity": "sha1-sdhVB9rzlkgo3lSzfQ1zumfdpWw=",
           "dev": true
         },
         "formatio": {
@@ -6000,7 +6262,7 @@
         "samsam": {
           "version": "1.3.0",
           "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz",
-          "integrity": "sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==",
+          "integrity": "sha1-jR2TUOJWItow3j5EumkrUiGrfFA=",
           "dev": true
         }
       }
@@ -6216,7 +6478,7 @@
     "split2": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz",
-      "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==",
+      "integrity": "sha1-GGsldbz4PoW30YRldWI47k7kJJM=",
       "dev": true,
       "requires": {
         "through2": "2.0.3"
@@ -6387,6 +6649,62 @@
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
       "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
     },
+    "superagent": {
+      "version": "3.8.2",
+      "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.2.tgz",
+      "integrity": "sha512-gVH4QfYHcY3P0f/BZzavLreHW3T1v7hG9B+hpMQotGQqurOvhv87GcMCd6LWySmBuf+BDR44TQd0aISjVHLeNQ==",
+      "dev": true,
+      "requires": {
+        "component-emitter": "1.2.1",
+        "cookiejar": "2.1.2",
+        "debug": "3.1.0",
+        "extend": "3.0.1",
+        "form-data": "2.3.2",
+        "formidable": "1.2.1",
+        "methods": "1.1.2",
+        "mime": "1.6.0",
+        "qs": "6.5.1",
+        "readable-stream": "2.3.4"
+      },
+      "dependencies": {
+        "combined-stream": {
+          "version": "1.0.6",
+          "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz",
+          "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=",
+          "dev": true,
+          "requires": {
+            "delayed-stream": "1.0.0"
+          }
+        },
+        "form-data": {
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz",
+          "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=",
+          "dev": true,
+          "requires": {
+            "asynckit": "0.4.0",
+            "combined-stream": "1.0.6",
+            "mime-types": "2.1.17"
+          }
+        },
+        "mime": {
+          "version": "1.6.0",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+          "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+          "dev": true
+        }
+      }
+    },
+    "supertest": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/supertest/-/supertest-3.1.0.tgz",
+      "integrity": "sha512-O44AMnmJqx294uJQjfUmEyYOg7d9mylNFsMw/Wkz4evKd1njyPrtCN+U6ZIC7sKtfEVQhfTqFFijlXx8KP/Czw==",
+      "dev": true,
+      "requires": {
+        "methods": "1.1.2",
+        "superagent": "3.8.2"
+      }
+    },
     "supports-color": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.2.0.tgz",
@@ -6395,6 +6713,11 @@
         "has-flag": "3.0.0"
       }
     },
+    "sylvester": {
+      "version": "0.0.12",
+      "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.12.tgz",
+      "integrity": "sha1-WohEFc0tACxX56OqyZRip1zp/bQ="
+    },
     "table": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/table/-/table-3.8.3.tgz",
@@ -6488,13 +6811,13 @@
     "tcomb": {
       "version": "3.2.24",
       "resolved": "https://registry.npmjs.org/tcomb/-/tcomb-3.2.24.tgz",
-      "integrity": "sha512-N9IrL2iIyS/f4+WHYZaMh04ZqDL8yEit9cVdnn+fOuL6jbKo1fusNswHOjSo/kbYwLUKRS1OlQmAkyeNxyEUhA==",
+      "integrity": "sha1-f0JwU8w5O1mXxMPYWcogQRGAiHs=",
       "dev": true
     },
     "tcomb-validation": {
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/tcomb-validation/-/tcomb-validation-3.4.1.tgz",
-      "integrity": "sha512-urVVMQOma4RXwiVCa2nM2eqrAomHROHvWPuj6UkDGz/eb5kcy0x6P0dVt6kzpUZtYMNoAqJLWmz1BPtxrtjtrA==",
+      "integrity": "sha1-p2luwXbOVqCB2eAZ+LcypaiJS2U=",
       "dev": true,
       "requires": {
         "tcomb": "3.2.24"
@@ -6509,7 +6832,7 @@
     "text-extensions": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.7.0.tgz",
-      "integrity": "sha512-AKXZeDq230UaSzaO5s3qQUZOaC7iKbzq0jOFL614R7d9R593HLqAOL0cYoqLdkNrjBSOdmoQI06yigq1TSBXAg==",
+      "integrity": "sha1-+qq6JiXtdG1WiiPk0KrNm/CKizk=",
       "dev": true
     },
     "text-table": {
@@ -6647,7 +6970,7 @@
     "type-detect": {
       "version": "4.0.8",
       "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
-      "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
+      "integrity": "sha1-dkb7XxiHHPu3dJ5pvTmmOI63RQw=",
       "dev": true
     },
     "type-is": {
@@ -6682,7 +7005,7 @@
     "ua-parser-js": {
       "version": "0.7.17",
       "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.17.tgz",
-      "integrity": "sha512-uRdSdu1oA1rncCQL7sCj8vSyZkgtL7faaw9Tc9rZ3mGgraQ7+Pdx7w5mnOSF3gw9ZNG6oc+KXfkon3bKuROm0g==",
+      "integrity": "sha1-6exflJi57JEOeuOsYmqAXE0J7Kw=",
       "dev": true
     },
     "uglify-js": {

From d74df56035beea04d6fd438b82f1adef32ca2e20 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Aug 2018 11:36:22 +0100
Subject: [PATCH 30/33] chore: update config docs

---
 config.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.js b/config.js
index 542432d5..7a7e60bf 100755
--- a/config.js
+++ b/config.js
@@ -135,7 +135,7 @@ var conf = convict({
   },
   search: {
     enabled: {
-      doc: 'If enabled, search will index content',
+      doc: 'If true, API responds to collection /search endpoints',
       format: Boolean,
       default: false
     },

From 214d49524dcee8b7ac30eeb386ab60d97e6f8aad Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Aug 2018 11:42:18 +0100
Subject: [PATCH 31/33] chore: update data connector requirement

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 1eb852f5..a57c69c9 100644
--- a/package.json
+++ b/package.json
@@ -88,7 +88,7 @@
     "uuid": "^3.3.2"
   },
   "dataConnectorDependencies": {
-    "@dadi/api-mongodb": "4.1.0"
+    "@dadi/api-mongodb": "4.2.0"
   },
   "greenkeeper": {
     "ignore": [

From d5949a09dcf806f040af6cad715b56151f01a094 Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Aug 2018 11:42:35 +0100
Subject: [PATCH 32/33] refactor: slight logic modifications

---
 dadi/lib/controller/searchIndex.js | 2 +-
 dadi/lib/search/index.js           | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/dadi/lib/controller/searchIndex.js b/dadi/lib/controller/searchIndex.js
index 363669da..f1c5db3c 100644
--- a/dadi/lib/controller/searchIndex.js
+++ b/dadi/lib/controller/searchIndex.js
@@ -18,7 +18,7 @@ SearchIndex.prototype.post = function (req, res, next) {
   }
 
   // 404 if Search is not enabled
-  if (config.get('search.enabled') === false) {
+  if (config.get('search.enabled') !== true) {
     return next()
   }
 
diff --git a/dadi/lib/search/index.js b/dadi/lib/search/index.js
index f19aae59..ac869351 100644
--- a/dadi/lib/search/index.js
+++ b/dadi/lib/search/index.js
@@ -154,11 +154,7 @@ Search.prototype.find = function (searchTerm) {
  * @return {Promise} - Query to delete instances with matching document ids.
  */
 Search.prototype.delete = function (documents) {
-  if (!this.canUse()) {
-    return Promise.resolve()
-  }
-
-  if (!Array.isArray(documents)) {
+  if (!this.canUse() || !Array.isArray(documents)) {
     return Promise.resolve()
   }
 

From 2ae7ac26b45127a4f458b09cf3fa43c3d1152bfe Mon Sep 17 00:00:00 2001
From: James Lambie <jameslambie@gmail.com>
Date: Wed, 1 Aug 2018 11:46:20 +0100
Subject: [PATCH 33/33] chore: move metadata dependency

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index a57c69c9..098518b9 100644
--- a/package.json
+++ b/package.json
@@ -23,6 +23,7 @@
     "@dadi/et": "^2.0.0",
     "@dadi/format-error": "^1.7.0",
     "@dadi/logger": "^1.4.1",
+    "@dadi/metadata": "^2.0.0",
     "@dadi/status": "latest",
     "async": "^2.6.1",
     "aws-sdk": "2.249.1",
@@ -65,7 +66,6 @@
   "devDependencies": {
     "@commitlint/cli": "~4.1.1",
     "@commitlint/config-angular": "~3.1.1",
-    "@dadi/metadata": "^2.0.0",
     "aws-sdk-mock": "1.6.1",
     "coveralls": "^3.0.1",
     "env-test": "1.0.0",