Monitoring telemetry collection interval

src/legacy/server/usage/classes/__tests__/collector_set.js

@@ -22,6 +22,7 @@ import sinon from 'sinon';
 import expect from '@kbn/expect';
 import { Collector } from '../collector';
 import { CollectorSet } from '../collector_set';
+import { UsageCollector } from '../usage_collector';
 
 describe('CollectorSet', () => {
   describe('registers a collector set and runs lifecycle events', () => {
@@ -140,6 +141,26 @@ describe('CollectorSet', () => {
       });
     });
   });
+
+  describe('isUsageCollector', () => {
+    const server = { };
+    const collectorOptions = { type: 'MY_TEST_COLLECTOR', fetch: () => {} };
+
+    it('returns true only for UsageCollector instances', () => {
+      const collectors = new CollectorSet(server);
+
+      const usageCollector = new UsageCollector(server, collectorOptions);
+      const collector = new Collector(server, collectorOptions);
+      const randomClass = new (class Random {});
+      expect(collectors.isUsageCollector(usageCollector)).to.be(true);
+      expect(collectors.isUsageCollector(collector)).to.be(false);
+      expect(collectors.isUsageCollector(randomClass)).to.be(false);
+      expect(collectors.isUsageCollector({})).to.be(false);
+      expect(collectors.isUsageCollector(null)).to.be(false);
+      expect(collectors.isUsageCollector('')).to.be(false);
+      expect(collectors.isUsageCollector()).to.be(false);
+    });
+  });
 });
 
 

src/legacy/server/usage/classes/collector_set.js

@@ -68,6 +68,11 @@ export class CollectorSet {
     return this._collectors.find(c => c.type === type);
   }
 
+  // isUsageCollector(x: UsageCollector | any): x is UsageCollector {
+  isUsageCollector(x) {
+    return x instanceof UsageCollector;
+  }
+
   /*
    * Call a bunch of fetch methods and then do them in bulk
    * @param {CollectorSet} collectorSet - a set of collectors to fetch. Default to all registered collectors

x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js

@@ -17,6 +17,9 @@ class MockCollectorSet {
     this.mockServer = _mockServer;
     this.mockCollectors = mockCollectors;
   }
+  isUsageCollector(x) {
+    return !!x.isUsageCollector;
+  }
   getCollectorByType(type) {
     return this.mockCollectors.find(collector => collector.type === type) || this.mockCollectors[0];
   }
@@ -41,6 +44,9 @@ describe('BulkUploader', () => {
       server = {
         log: sinon.spy(),
         plugins: {
+          xpack_main: {
+            telemetryCollectionInterval: 3000,
+          },
           elasticsearch: {
             createCluster: () => cluster,
             getCluster: () => cluster,
@@ -121,5 +127,67 @@ describe('BulkUploader', () => {
         done();
       }, CHECK_DELAY);
     });
+
+    it('does not call UsageCollectors if last reported is within the usageInterval', done => {
+      const usageCollectorFetch = sinon.stub();
+      const collectorFetch = sinon.stub().returns({ type: 'type_usage_collector_test', result: { testData: 12345 } });
+
+      const collectors = new MockCollectorSet(server, [
+        {
+          fetch: usageCollectorFetch,
+          formatForBulkUpload: result => result,
+          isUsageCollector: true,
+        },
+        {
+          fetch: collectorFetch,
+          formatForBulkUpload: result => result,
+          isUsageCollector: false,
+        }
+      ]);
+
+      const uploader = new BulkUploader(server, {
+        interval: FETCH_INTERVAL
+      });
+      uploader._lastFetchUsageTime = Date.now();
+
+      uploader.start(collectors);
+      setTimeout(() => {
+        uploader.stop();
+        expect(collectorFetch.callCount).to.be.greaterThan(0);
+        expect(usageCollectorFetch.callCount).to.eql(0);
+        done();
+      }, CHECK_DELAY);
+    });
+
+    it('calls UsageCollectors if last reported exceeds during a _usageInterval', done => {
+      const usageCollectorFetch = sinon.stub();
+      const collectorFetch = sinon.stub().returns({ type: 'type_usage_collector_test', result: { testData: 12345 } });
+
+      const collectors = new MockCollectorSet(server, [
+        {
+          fetch: usageCollectorFetch,
+          formatForBulkUpload: result => result,
+          isUsageCollector: true,
+        },
+        {
+          fetch: collectorFetch,
+          formatForBulkUpload: result => result,
+          isUsageCollector: false,
+        }
+      ]);
+
+      const uploader = new BulkUploader(server, {
+        interval: FETCH_INTERVAL
+      });
+      uploader._lastFetchUsageTime = Date.now() - uploader._usageInterval;
+
+      uploader.start(collectors);
+      setTimeout(() => {
+        uploader.stop();
+        expect(collectorFetch.callCount).to.be.greaterThan(0);
+        expect(usageCollectorFetch.callCount).to.be.greaterThan(0);
+        done();
+      }, CHECK_DELAY);
+    });
   });
 });

x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js

@@ -6,6 +6,7 @@
 
 import { defaultsDeep, uniq, compact } from 'lodash';
 import { callClusterFactory } from '../../../xpack_main';
+
 import {
   LOGGING_TAG,
   KIBANA_MONITORING_LOGGING_TAG,
@@ -42,6 +43,9 @@ export class BulkUploader {
 
     this._timer = null;
     this._interval = interval;
+    this._lastFetchUsageTime = null;
+    this._usageInterval = server.plugins.xpack_main.telemetryCollectionInterval;
+
     this._log = {
       debug: message => server.log(['debug', ...LOGGING_TAGS], message),
       info: message => server.log(['info', ...LOGGING_TAGS], message),
@@ -63,18 +67,33 @@ export class BulkUploader {
    */
   start(collectorSet) {
     this._log.info('Starting monitoring stats collection');
+    const filterCollectorSet = _collectorSet => {
+      const filterUsage = this._lastFetchUsageTime && this._lastFetchUsageTime + this._usageInterval > Date.now();
+      if (!filterUsage) {
+        this._lastFetchUsageTime = Date.now();
+      }
 
-    // this is internal bulk upload, so filter out API-only collectors
-    const filterThem = _collectorSet => _collectorSet.getFilteredCollectorSet(c => c.ignoreForInternalUploader !== true);
+      return _collectorSet.getFilteredCollectorSet(c => {
+        // this is internal bulk upload, so filter out API-only collectors
+        if (c.ignoreForInternalUploader) {
+          return false;
+        }
+        // Only collect usage data at the same interval as telemetry would (default to once a day)
+        if (filterUsage && _collectorSet.isUsageCollector(c)) {
+          return false;
+        }
+        return true;
+      });
+    };
 
     if (this._timer) {
       clearInterval(this._timer);
     } else {
-      this._fetchAndUpload(filterThem(collectorSet)); // initial fetch
+      this._fetchAndUpload(filterCollectorSet(collectorSet)); // initial fetch
     }
 
     this._timer = setInterval(() => {
-      this._fetchAndUpload(filterThem(collectorSet));
+      this._fetchAndUpload(filterCollectorSet(collectorSet));
     }, this._interval);
   }
 

x-pack/plugins/xpack_main/server/lib/setup_xpack_main.js

@@ -7,6 +7,7 @@
 import { injectXPackInfoSignature } from './inject_xpack_info_signature';
 import { XPackInfo } from './xpack_info';
 import { FeatureRegistry } from './feature_registry';
+import { REPORT_INTERVAL_MS } from '../../common/constants';
 
 /**
  * Setup the X-Pack Main plugin. This is fired every time that the Elasticsearch plugin becomes Green.
@@ -22,6 +23,7 @@ export function setupXPackMain(server) {
   });
 
   server.expose('info', info);
+  server.expose('telemetryCollectionInterval', REPORT_INTERVAL_MS);
   server.expose('createXPackInfo', (options) => new XPackInfo(server, options));
   server.ext('onPreResponse', (request, h) => injectXPackInfoSignature(info, request, h));
 

x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_high_level_stats.js

@@ -159,6 +159,20 @@ function getIndexPatternForStackProduct(product) {
   return null;
 }
 
+/**
+ * Returns additional, product-specific, query filters
+ *
+ * @param {*} product The product id, which should be in the constants file
+ * @param {Array} array of filter clauses
+ */
+function getFiltersForStackProduct(product) {
+  switch (product) {
+    case KIBANA_SYSTEM_ID:
+      return [ { exists: { field: 'kibana_stats.usage.index' } } ];
+  }
+  return [];
+}
+
 /**
  * Get statistics about selected Elasticsearch clusters, for the selected {@code product}.
  *
@@ -167,7 +181,7 @@ function getIndexPatternForStackProduct(product) {
  * @param {Array} clusterUuids The string Cluster UUIDs to fetch details for
  * @param {Date} start Start time to limit the stats
  * @param {Date} end End time to limit the stats
- * @param {String} product The product to limit too ('kibana', 'logstash', 'beats')
+ * @param {String} product The product to limit to ('kibana', 'logstash', 'beats')
  * @return {Promise} Object keyed by the cluster UUIDs to make grouping easier.
  */
 export function getHighLevelStats(server, callCluster, clusterUuids, start, end, product) {
@@ -184,7 +198,7 @@ export function getHighLevelStats(server, callCluster, clusterUuids, start, end,
  * @param {Array} clusterUuids Cluster UUIDs to limit the request against
  * @param {Date} start Start time to limit the stats
  * @param {Date} end End time to limit the stats
- * @param {String} product The product to limit too ('kibana', 'logstash', 'beats')
+ * @param {String} product The product to limit to ('kibana', 'logstash', 'beats')
  * @return {Promise} Response for the instances to fetch detailed for the product.
  */
 export function fetchHighLevelStats(server, callCluster, clusterUuids, start, end, product) {
@@ -210,7 +224,10 @@ export function fetchHighLevelStats(server, callCluster, clusterUuids, start, en
         start,
         end,
         type: `${product}_stats`,
-        filters: [ { terms: { cluster_uuid: clusterUuids } } ]
+        filters: [
+          { terms: { cluster_uuid: clusterUuids } },
+          ...getFiltersForStackProduct(product)
+        ]
       }),
       collapse: {
         // a more ideal field would be the concatenation of the uuid + transport address for duped UUIDs (copied installations)

x-pack/test/functional/es_archives/monitoring/basic_6.3.x/mappings.json

@@ -4,7 +4,7 @@
     "aliases": {
     },
     "index": ".monitoring-kibana-6-2018.07.23",
-    "mappings":{"_doc":{"dynamic":false,"properties":{"cluster_uuid":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"},"interval_ms":{"type":"long"},"type":{"type":"keyword"},"source_node":{"properties":{"uuid":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"ip":{"type":"keyword"},"name":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"}}},"kibana_stats":{"properties":{"kibana":{"properties":{"uuid":{"type":"keyword"},"name":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"version":{"type":"keyword"},"snapshot":{"type":"boolean"},"status":{"type":"keyword"},"statuses":{"properties":{"name":{"type":"keyword"},"state":{"type":"keyword"}}}}},"cloud":{"properties":{"name":{"type":"keyword"},"id":{"type":"keyword"},"vm_type":{"type":"keyword"},"region":{"type":"keyword"},"zone":{"type":"keyword"},"metadata":{"type":"object"}}},"os":{"properties":{"load":{"properties":{"1m":{"type":"half_float"},"5m":{"type":"half_float"},"15m":{"type":"half_float"}}},"memory":{"properties":{"total_in_bytes":{"type":"float"},"free_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"}}},"uptime_in_millis":{"type":"long"}}},"process":{"properties":{"memory":{"properties":{"heap":{"properties":{"total_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"},"size_limit":{"type":"float"}}},"resident_set_size_in_bytes":{"type":"float"}}},"event_loop_delay":{"type":"float"},"uptime_in_millis":{"type":"long"}}},"sockets":{"properties":{"http":{"properties":{"total":{"type":"long"}}},"https":{"properties":{"total":{"type":"long"}}}}},"timestamp":{"type":"date"},"requests":{"properties":{"disconnects":{"type":"long"},"total":{"type":"long"},"status_codes":{"type":"object"}}},"response_times":{"properties":{"average":{"type":"float"},"max":{"type":"float"}}},"concurrent_connections":{"type":"long"}}}}}},
+    "mappings":{"_doc":{"dynamic":false,"properties":{"cluster_uuid":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"},"interval_ms":{"type":"long"},"type":{"type":"keyword"},"source_node":{"properties":{"uuid":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"ip":{"type":"keyword"},"name":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"}}},"kibana_stats":{"properties":{"usage":{"properties":{"index":{"type":"keyword"}}},"kibana":{"properties":{"uuid":{"type":"keyword"},"name":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"version":{"type":"keyword"},"snapshot":{"type":"boolean"},"status":{"type":"keyword"},"statuses":{"properties":{"name":{"type":"keyword"},"state":{"type":"keyword"}}}}},"cloud":{"properties":{"name":{"type":"keyword"},"id":{"type":"keyword"},"vm_type":{"type":"keyword"},"region":{"type":"keyword"},"zone":{"type":"keyword"},"metadata":{"type":"object"}}},"os":{"properties":{"load":{"properties":{"1m":{"type":"half_float"},"5m":{"type":"half_float"},"15m":{"type":"half_float"}}},"memory":{"properties":{"total_in_bytes":{"type":"float"},"free_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"}}},"uptime_in_millis":{"type":"long"}}},"process":{"properties":{"memory":{"properties":{"heap":{"properties":{"total_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"},"size_limit":{"type":"float"}}},"resident_set_size_in_bytes":{"type":"float"}}},"event_loop_delay":{"type":"float"},"uptime_in_millis":{"type":"long"}}},"sockets":{"properties":{"http":{"properties":{"total":{"type":"long"}}},"https":{"properties":{"total":{"type":"long"}}}}},"timestamp":{"type":"date"},"requests":{"properties":{"disconnects":{"type":"long"},"total":{"type":"long"},"status_codes":{"type":"object"}}},"response_times":{"properties":{"average":{"type":"float"},"max":{"type":"float"}}},"concurrent_connections":{"type":"long"}}}}}},
     "settings": {
       "index": {
         "auto_expand_replicas": "0-1",

x-pack/test/functional/es_archives/monitoring/multicluster/mappings.json

@@ -42,7 +42,7 @@
     "aliases": {
     },
     "index": ".monitoring-kibana-6-2017.08.15",
-    "mappings":{"_doc":{"dynamic":false,"properties":{"cluster_uuid":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"},"interval_ms":{"type":"long"},"type":{"type":"keyword"},"source_node":{"properties":{"uuid":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"ip":{"type":"keyword"},"name":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"}}},"kibana_stats":{"properties":{"kibana":{"properties":{"uuid":{"type":"keyword"},"name":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"version":{"type":"keyword"},"snapshot":{"type":"boolean"},"status":{"type":"keyword"},"statuses":{"properties":{"name":{"type":"keyword"},"state":{"type":"keyword"}}}}},"cloud":{"properties":{"name":{"type":"keyword"},"id":{"type":"keyword"},"vm_type":{"type":"keyword"},"region":{"type":"keyword"},"zone":{"type":"keyword"},"metadata":{"type":"object"}}},"os":{"properties":{"load":{"properties":{"1m":{"type":"half_float"},"5m":{"type":"half_float"},"15m":{"type":"half_float"}}},"memory":{"properties":{"total_in_bytes":{"type":"float"},"free_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"}}},"uptime_in_millis":{"type":"long"}}},"process":{"properties":{"memory":{"properties":{"heap":{"properties":{"total_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"},"size_limit":{"type":"float"}}},"resident_set_size_in_bytes":{"type":"float"}}},"event_loop_delay":{"type":"float"},"uptime_in_millis":{"type":"long"}}},"sockets":{"properties":{"http":{"properties":{"total":{"type":"long"}}},"https":{"properties":{"total":{"type":"long"}}}}},"timestamp":{"type":"date"},"requests":{"properties":{"disconnects":{"type":"long"},"total":{"type":"long"},"status_codes":{"type":"object"}}},"response_times":{"properties":{"average":{"type":"float"},"max":{"type":"float"}}},"concurrent_connections":{"type":"long"}}}}}},
+    "mappings":{"_doc":{"dynamic":false,"properties":{"cluster_uuid":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"},"interval_ms":{"type":"long"},"type":{"type":"keyword"},"source_node":{"properties":{"uuid":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"ip":{"type":"keyword"},"name":{"type":"keyword"},"timestamp":{"type":"date","format":"date_time"}}},"kibana_stats":{"properties":{"usage":{"properties":{"index":{"type":"keyword"}}},"kibana":{"properties":{"uuid":{"type":"keyword"},"name":{"type":"keyword"},"host":{"type":"keyword"},"transport_address":{"type":"keyword"},"version":{"type":"keyword"},"snapshot":{"type":"boolean"},"status":{"type":"keyword"},"statuses":{"properties":{"name":{"type":"keyword"},"state":{"type":"keyword"}}}}},"cloud":{"properties":{"name":{"type":"keyword"},"id":{"type":"keyword"},"vm_type":{"type":"keyword"},"region":{"type":"keyword"},"zone":{"type":"keyword"},"metadata":{"type":"object"}}},"os":{"properties":{"load":{"properties":{"1m":{"type":"half_float"},"5m":{"type":"half_float"},"15m":{"type":"half_float"}}},"memory":{"properties":{"total_in_bytes":{"type":"float"},"free_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"}}},"uptime_in_millis":{"type":"long"}}},"process":{"properties":{"memory":{"properties":{"heap":{"properties":{"total_in_bytes":{"type":"float"},"used_in_bytes":{"type":"float"},"size_limit":{"type":"float"}}},"resident_set_size_in_bytes":{"type":"float"}}},"event_loop_delay":{"type":"float"},"uptime_in_millis":{"type":"long"}}},"sockets":{"properties":{"http":{"properties":{"total":{"type":"long"}}},"https":{"properties":{"total":{"type":"long"}}}}},"timestamp":{"type":"date"},"requests":{"properties":{"disconnects":{"type":"long"},"total":{"type":"long"},"status_codes":{"type":"object"}}},"response_times":{"properties":{"average":{"type":"float"},"max":{"type":"float"}}},"concurrent_connections":{"type":"long"}}}}}},
     "settings": {
       "index": {
         "auto_expand_replicas": "0-1",