Use scroll API when exporting all

Fixes #5586
elastic · Dec 9, 2015 · 142800c · 142800c
1 parent 4d9ddac
commit 142800c
Show file tree

Hide file tree

Showing 5 changed files with 162 additions and 34 deletions.
diff --git a/src/plugins/elasticsearch/index.js b/src/plugins/elasticsearch/index.js
@@ -38,6 +38,7 @@ module.exports = function (kibana) {
       createProxy(server, 'POST', '/{index}/_search');
       createProxy(server, 'POST', '/{index}/_field_stats');
       createProxy(server, 'POST', '/_msearch');
+      createProxy(server, 'POST', '/_search/scroll');
 
       function noBulkCheck(request, reply) {
         if (/\/_bulk/.test(request.path)) {

diff --git a/src/plugins/kibana/public/dashboard/services/saved_dashboards.js b/src/plugins/kibana/public/dashboard/services/saved_dashboards.js
@@ -41,6 +41,53 @@ define(function (require) {
       });
     };
 
+    this.scan = function (pageSize = 100, docCount = 1000) {
+      var allResults = {
+        hits: [],
+        total: 0
+      };
+
+      var self = this;
+      return new Promise(function (resolve, reject) {
+        es.search({
+          index: kbnIndex,
+          type: 'dashboard',
+          size: pageSize,
+          body: { query: {match_all: {}}},
+          searchType: 'scan',
+          scroll: '1m'
+        }, function getMoreUntilDone(error, response) {
+          var scanAllResults = docCount === Number.POSITIVE_INFINITY;
+          allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, docCount);
+
+          var hits = response.hits.hits
+          .slice(0, allResults.total - allResults.hits.length)
+          .map(self.mapHits.bind(self));
+          allResults.hits =  allResults.hits.concat(hits);
+
+          var collectedAllResults = allResults.total === allResults.hits.length;
+          if (collectedAllResults) {
+            resolve(allResults);
+          } else {
+            es.scroll({
+              scrollId: response._scroll_id,
+            }, getMoreUntilDone);
+          }
+        });
+      });
+    };
+
+    this.scanAll = function (queryString, pageSize = 100) {
+      return this.scan(pageSize, Number.POSITIVE_INFINITY);
+    };
+
+    this.mapHits = function (hit) {
+      var source = hit._source;
+      source.id = hit._id;
+      source.url = this.urlFor(hit._id);
+      return source;
+    };
+
     this.find = function (searchString, size = 100) {
       var self = this;
       var body;
@@ -67,12 +114,7 @@ define(function (require) {
       .then(function (resp) {
         return {
           total: resp.hits.total,
-          hits: resp.hits.hits.map(function (hit) {
-            var source = hit._source;
-            source.id = hit._id;
-            source.url = self.urlFor(hit._id);
-            return source;
-          })
+          hits: resp.hits.hits.map(self.mapHits.bind(self))
         };
       });
     };

diff --git a/src/plugins/kibana/public/discover/saved_searches/saved_searches.js b/src/plugins/kibana/public/discover/saved_searches/saved_searches.js
@@ -31,6 +31,47 @@ define(function (require) {
       nouns: 'saved searches'
     };
 
+    this.scan = function (pageSize = 100, docCount = 1000) {
+      var allResults = {
+        hits: [],
+        total: 0
+      };
+
+      var self = this;
+      return new Promise(function (resolve, reject) {
+        es.search({
+          index: kbnIndex,
+          type: 'search',
+          size: pageSize,
+          body: { query: {match_all: {}}},
+          searchType: 'scan',
+          scroll: '1m'
+        }, function getMoreUntilDone(error, response) {
+          var scanAllResults = docCount === Number.POSITIVE_INFINITY;
+          allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, docCount);
+
+          var hits = response.hits.hits
+          .slice(0, allResults.total - allResults.hits.length)
+          .map(self.mapHits.bind(self));
+          allResults.hits =  allResults.hits.concat(hits);
+
+          var collectedAllResults = allResults.total === allResults.hits.length;
+          if (collectedAllResults) {
+            resolve(allResults);
+          } else {
+            es.scroll({
+              scrollId: response._scroll_id,
+            }, getMoreUntilDone);
+          }
+        });
+      });
+    };
+
+    this.scanAll = function (queryString, pageSize = 100) {
+      return this.scan(pageSize, Number.POSITIVE_INFINITY);
+    };
+
+
     this.get = function (id) {
       return (new SavedSearch(id)).init();
     };
@@ -46,6 +87,13 @@ define(function (require) {
       });
     };
 
+    this.mapHits = function (hit) {
+      var source = hit._source;
+      source.id = hit._id;
+      source.url = this.urlFor(hit._id);
+      return source;
+    };
+
     this.find = function (searchString, size = 100) {
       var self = this;
       var body;
@@ -72,12 +120,7 @@ define(function (require) {
       .then(function (resp) {
         return {
           total: resp.hits.total,
-          hits: resp.hits.hits.map(function (hit) {
-            var source = hit._source;
-            source.id = hit._id;
-            source.url = self.urlFor(hit._id);
-            return source;
-          })
+          hits: resp.hits.hits.map(self.mapHits.bind(self))
         };
       });
     };

diff --git a/src/plugins/kibana/public/settings/sections/objects/_objects.js b/src/plugins/kibana/public/settings/sections/objects/_objects.js
@@ -94,7 +94,7 @@ define(function (require) {
 
         $scope.exportAll = () => {
           Promise.map($scope.services, (service) =>
-            service.service.find('', MAX_SIZE).then((results) =>
+            service.service.scanAll('').then((results) =>
               results.hits.map((hit) => _.extend(hit, {type: service.type}))
             )
           ).then((results) => retrieveAndExportDocs(_.flattenDeep(results)));

diff --git a/src/plugins/kibana/public/visualize/saved_visualizations/saved_visualizations.js b/src/plugins/kibana/public/visualize/saved_visualizations/saved_visualizations.js
@@ -41,6 +41,68 @@ define(function (require) {
       });
     };
 
+    this.scan = function (pageSize = 100, docCount = 1000) {
+      var allResults = {
+        hits: [],
+        total: 0
+      };
+
+      var self = this;
+      return new Promise(function (resolve, reject) {
+        es.search({
+          index: kbnIndex,
+          type: 'visualization',
+          size: pageSize,
+          body: { query: {match_all: {}}},
+          searchType: 'scan',
+          scroll: '1m'
+        }, function getMoreUntilDone(error, response) {
+          var scanAllResults = docCount === Number.POSITIVE_INFINITY;
+          allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, docCount);
+
+          var hits = response.hits.hits
+          .slice(0, allResults.total - allResults.hits.length)
+          .map(self.mapHits.bind(self));
+          allResults.hits =  allResults.hits.concat(hits);
+
+          var collectedAllResults = allResults.total === allResults.hits.length;
+          if (collectedAllResults) {
+            resolve(allResults);
+          } else {
+            es.scroll({
+              scrollId: response._scroll_id,
+            }, getMoreUntilDone);
+          }
+        });
+      });
+    };
+
+    this.scanAll = function (queryString, pageSize = 100) {
+      return this.scan(pageSize, Number.POSITIVE_INFINITY);
+    };
+
+    this.mapHits = function (hit) {
+      var source = hit._source;
+      source.id = hit._id;
+      source.url = this.urlFor(hit._id);
+
+      var typeName = source.typeName;
+      if (source.visState) {
+        try { typeName = JSON.parse(source.visState).type; }
+        catch (e) { /* missing typename handled below */ } // eslint-disable-line no-empty
+      }
+
+      if (!typeName || !visTypes.byName[typeName]) {
+        if (!typeName) notify.error('Visualization type is missing. Please add a type to this visualization.', hit);
+        else notify.error('Visualization type of "' + typeName + '" is invalid. Please change to a valid type.', hit);
+        return kbnUrl.redirect('/settings/objects/savedVisualizations/{{id}}', {id: source.id});
+      }
+
+      source.type = visTypes.byName[typeName];
+      source.icon = source.type.icon;
+      return source;
+    };
+
     this.find = function (searchString, size = 100) {
       var self = this;
       var body;
@@ -67,27 +129,7 @@ define(function (require) {
       .then(function (resp) {
         return {
           total: resp.hits.total,
-          hits: _.transform(resp.hits.hits, function (hits, hit) {
-            var source = hit._source;
-            source.id = hit._id;
-            source.url = self.urlFor(hit._id);
-
-            var typeName = source.typeName;
-            if (source.visState) {
-              try { typeName = JSON.parse(source.visState).type; }
-              catch (e) { /* missing typename handled below */ } // eslint-disable-line no-empty
-            }
-
-            if (!typeName || !visTypes.byName[typeName]) {
-              if (!typeName) notify.error('Visualization type is missing. Please add a type to this visualization.', hit);
-              else notify.error('Visualization type of "' + typeName + '" is invalid. Please change to a valid type.', hit);
-              return kbnUrl.redirect('/settings/objects/savedVisualizations/{{id}}', {id: source.id});
-            }
-
-            source.type = visTypes.byName[typeName];
-            source.icon = source.type.icon;
-            hits.push(source);
-          }, [])
+          hits: resp.hits.hits.map(self.mapHits.bind(self))
         };
       });
     };