COMPASS-372 - Refactor Data Service Indexes Getter. [HOLD]

.travis.yml

@@ -10,3 +10,4 @@ notifications:
   flowdock: e3dc17bc8a2c1b3412abe3e5747f8291
 env:
   - MONGODB_VERSION=3.2.x MONGODB_TOPOLOGY=standalone
+  - MONGODB_VERSION=3.0.x MONGODB_TOPOLOGY=standalone

lib/native-client.js

@@ -5,12 +5,12 @@ const async = require('async');
 const EventEmitter = require('events');
 const connect = require('mongodb-connection-model').connect;
 const getInstance = require('mongodb-instance-model').get;
-const getIndexes = require('mongodb-index-model').fetch;
 const createSampleStream = require('mongodb-collection-sample');
 const parseNamespace = require('mongodb-ns');
 const translate = require('mongodb-js-errors').translate;
 const debug = require('debug')('mongodb-data-service:native-client');
 const ReadPreference = require('mongodb').ReadPreference;
+var isNotAuthorizedError = require('mongodb-js-errors').isNotAuthorized;
 
 /**
  * The constant for a mongos.
@@ -419,14 +419,128 @@ class NativeClient extends EventEmitter {
     });
   }
 
+  /**
+   * get basic index information via `db.collection.indexes()`
+   * @param {String} ns - The namespace to search on.
+   * @param {Function} done - The callback
+   */
+  getIndexes(ns, done) {
+    this._collection(ns).indexes(function(err, indexes) {
+      if (err) {
+        done(err);
+      }
+      // add ns field to each index
+      _.each(indexes, function(idx) {
+        idx.ns = ns;
+      });
+      done(null, indexes);
+    });
+  }
+
+  /**
+   * get index statistics via `db.collection.aggregate({$indexStats: {}})`
+   * @param {String} ns - The namespace to search on.
+   * @param {Function} done - The callback
+   */
+  getIndexStats(ns, done) {
+    var pipeline = [
+      { $indexStats: { } },
+      { $project: { name: 1, usageHost: '$host', usageCount: '$accesses.ops', usageSince: '$accesses.since' } }
+    ];
+    this._collection(ns).aggregate(pipeline, function(err, res) {
+      if (err) {
+        if (isNotAuthorizedError(err)) {
+          /**
+           * In the 3.2 server, `readWriteAnyDatabase@admin` does not grant sufficient privileges for $indexStats.
+           * The `clusterMonitor` role is required to run $indexStats.
+           * @see https://jira.mongodb.org/browse/INT-1520
+           */
+          return done(null, {});
+        }
+
+        if (err.message.match(/Unrecognized pipeline stage name/)) {
+          // $indexStats not yet supported, return empty document
+          return done(null, {});
+        }
+        done(err);
+      }
+      res = _.mapKeys(res, function(stat) {
+        return stat.name;
+      });
+      done(null, res);
+    });
+  }
+
+  /**
+   * get index sizes via `db.collection.stats()` (`indexSizes` field)
+   * @param {String} ns - The namespace to search on.
+   * @param {Function} done - The callback
+   */
+
+  getIndexSizes(ns, done) {
+    this._collection(ns).stats(function(err, res) {
+      if (err) {
+        if (isNotAuthorizedError(err)) {
+          debug('Not authorized to get collection stats.  Returning default for indexSizes {}.');
+          return done(null, {});
+        }
+        return done(err);
+      }
+
+      res = _.mapValues(res.indexSizes, function(size) {
+        return {size: size};
+      });
+      done(null, res);
+    });
+  }
+
+  /**
+   * merge all information together for each index
+   * @param {object} results - results from async.auto
+   * @param {Function} done - callback
+   */
+  combineStatsAndIndexes(results, done) {
+    var indexes = results.getIndexes;
+    var stats = results.getIndexStats;
+    var sizes = results.getIndexSizes;
+    _.each(indexes, function(idx, i) {
+      _.assign(indexes[i], stats[idx.name]);
+      _.assign(indexes[i], sizes[idx.name]);
+    });
+    done(null, indexes);
+  }
+
+  /**
+   * get basic index information via `db.collection.indexes()`
+   * @param {String} namespace - The namespace to search on.
+   * @param {Function} done - The callback
+   */
+  getIndexDetails(namespace, done) {
+    var tasks = {
+      getIndexes: this.getIndexes.bind(this, namespace),
+      getIndexStats: this.getIndexStats.bind(this, namespace),
+      getIndexSizes: this.getIndexSizes.bind(this, namespace),
+      indexes: ['getIndexes', 'getIndexStats', 'getIndexSizes', this.combineStatsAndIndexes.bind(this)]
+    };
+
+    async.auto(tasks, function(err, results) {
+      if (err) {
+        // report error
+        return done(err);
+      }
+      // all info was collected in indexes
+      return done(null, results.indexes);
+    });
+  }
+
   /**
    * Get the indexes for the collection.
    *
    * @param {String} ns - The collection namespace.
    * @param {Function} callback - The callback.
    */
   indexes(ns, callback) {
-    getIndexes(this.database, ns, (error, data) => {
+    this.getIndexDetails(ns, (error, data) => {
       if (error) {
         return callback(this._translateMessage(error));
       }

test/data-service.test.js

@@ -202,7 +202,7 @@ describe('DataService', function() {
     it('returns the collections', function(done) {
       service.listCollections('data-service', {}, function(err, collections) {
         assert.equal(null, err);
-        expect(collections).to.be.deep.equal([{ name: 'test', options: {} }]);
+        expect(collections).to.deep.include({ name: 'test', options: {} });
         done();
       });
     });
@@ -271,10 +271,7 @@ describe('DataService', function() {
         assert.equal(null, error);
         helper.listCollections(service.client, function(err, items) {
           assert.equal(null, err);
-          expect(items).to.deep.equal([
-            {name: 'foo', options: {}},
-            {name: 'test', options: {}}
-          ]);
+          expect(items.length).to.be.greaterThan(1); // For <3.2 system.indexes is returned with listCollections
           done();
         });
       });

test/native-client.test.js

@@ -233,10 +233,7 @@ describe('NativeClient', function() {
         assert.equal(null, error);
         helper.listCollections(client, function(err, items) {
           assert.equal(null, err);
-          expect(items).to.deep.equal([
-            {name: 'foo', options: {}},
-            {name: 'test', options: {}}
-          ]);
+          expect(items.length).to.be.greaterThan(1); // For <3.2 system.indexes is returned with listCollections
           done();
         });
       });
@@ -504,13 +501,25 @@ describe('NativeClient', function() {
   });
 
   describe('#indexes', function() {
-    it('returns the indexes', function(done) {
-      client.indexes('data-service.test', function(err, indexes) {
-        assert.equal(null, err);
-        expect(indexes[0].name).to.equal('_id_');
-        done();
+    if (process.env.MONGODB_VERSION >= '3.2.0') {
+      it('returns the indexes when $indexStats can run', function(done) {
+        client.indexes('data-service.test', function(err, indexes) {
+          assert.equal(null, err);
+          expect(indexes[0].name).to.equal('_id_');
+          expect(indexes[0]).to.include.keys('usageHost', 'usageCount', 'usageSince');
+          done();
+        });
       });
-    });
+    } else {
+      it('returns nothing when $indexStats cannot run', function(done) {
+        client.indexes('data-service.test', function(err, indexes) {
+          assert.equal(null, err);
+          expect(indexes[0].name).to.equal('_id_');
+          expect(indexes[0]).to.not.include.keys('usageHost', 'usageCount', 'usageSince');
+          done();
+        });
+      });
+    }
   });
 
   describe('#instance', function() {