From abf23a54e9850aa868aeaf0e761ec63be57cb20a Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Fri, 20 Jul 2018 10:28:19 -0400 Subject: [PATCH 1/4] Add beats architecture stats to telemetry --- .../telemetry/monitoring/get_beats_stats.js | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js index 9b4b9733ee9538..18bbda73196a69 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js @@ -26,6 +26,10 @@ const getBaseStats = () => ({ count: 0, names: [] }, + architecture: { + count: 0, + names: [] + } }); @@ -37,7 +41,7 @@ const getBaseStats = () => ({ * @param {Object} clusterHostSets - the object keyed by cluster UUIDs to count the unique hosts * @param {Object} clusterModuleSets - the object keyed by cluster UUIDs to count the unique modules */ -export function processResults(results = [], { clusters, clusterHostSets, clusterInputSets, clusterModuleSets }) { +export function processResults(results = [], { clusters, clusterHostSets, clusterInputSets, clusterModuleSets, clusterArchitectureSets }) { const currHits = get(results, 'hits.hits', []); currHits.forEach(hit => { const clusterUuid = get(hit, '_source.cluster_uuid'); @@ -46,6 +50,7 @@ export function processResults(results = [], { clusters, clusterHostSets, cluste clusterHostSets[clusterUuid] = new Set(); clusterInputSets[clusterUuid] = new Set(); clusterModuleSets[clusterUuid] = new Set(); + clusterArchitectureSets[clusterUuid] = new Set(); } const processBeatsStatsResults = () => { @@ -97,6 +102,15 @@ export function processResults(results = [], { clusters, clusterHostSets, cluste clusters[clusterUuid].module.names = Array.from(moduleSet); clusters[clusterUuid].module.count += stateModule.count; } + + const stateHost = get(hit, '_source.beats_state.state.host'); + if (stateHost !== undefined) { + const hostSet = clusterArchitectureSets[clusterUuid]; + const hostArchPlatform = `${stateHost.architecture}/${stateHost.os.platform}`; + hostSet.add(hostArchPlatform); + clusters[clusterUuid].architecture.names = Array.from(hostSet); + clusters[clusterUuid].architecture.count += 1; + } }; if (get(hit, '_source.type') === 'beats_stats') { @@ -136,6 +150,7 @@ async function fetchBeatsByType(server, callCluster, clusterUuids, start, end, { 'hits.hits._source.beats_stats.metrics.libbeat.output.type', 'hits.hits._source.beats_state.state.input', 'hits.hits._source.beats_state.state.module', + 'hits.hits._source.beats_state.state.host', ], body: { query: createQuery({ @@ -194,7 +209,8 @@ export async function getBeatsStats(server, callCluster, clusterUuids, start, en clusters: {}, // the result object to be built up clusterHostSets: {}, // passed to processResults for tracking state in the results generation clusterInputSets: {}, // passed to processResults for tracking state in the results generation - clusterModuleSets: {} // passed to processResults for tracking state in the results generation + clusterModuleSets: {}, // passed to processResults for tracking state in the results generation + clusterArchitectureSets: {} // passed to processResults for tracking state in the results generation }; await Promise.all([ From 8eaf963ffc6923b7ca40cbaedd95961ecfd066ba Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Wed, 8 Aug 2018 11:35:58 -0400 Subject: [PATCH 2/4] Update tests --- .../fixtures/beats_stats_results.json | 20 +++++++++++++++++++ .../monitoring/__tests__/get_beats_stats.js | 13 ++++++++++++ 2 files changed, 33 insertions(+) diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/fixtures/beats_stats_results.json b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/fixtures/beats_stats_results.json index 108369dcba72a1..be6e4352514fe8 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/fixtures/beats_stats_results.json +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/fixtures/beats_stats_results.json @@ -34,6 +34,26 @@ } } }, + { + "_source": { + "type": "beats_state", + "cluster_uuid": "W7hppdX7R229Oy3KQbZrTw", + "beats_state": { + "state": { + "host": { + "name": "Elastic-MBP.local", + "architecture": "x86_64", + "os": { + "platform": "darwin", + "version": "10.13.6", + "family": "darwin", + "build": "17G65" + } + } + } + } + } + }, { "_source": { "type": "beats_stats", diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js index cbd9da52b44909..d1ea9214b1642b 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js @@ -14,6 +14,7 @@ const getBaseOptions = () => ({ clusterHostSets: {}, clusterInputSets: {}, clusterModuleSets: {}, + clusterArchitectureSets: {} }); describe('Get Beats Stats', () => { @@ -111,6 +112,10 @@ describe('Get Beats Stats', () => { count: 0, names: [], }, + architecture: { + count: 0, + names: [] + } }, }); }); @@ -140,6 +145,10 @@ describe('Get Beats Stats', () => { count: 1, names: [ 'firehose' ], }, + architecture: { + count: 1, + names: [ 'x86_64/darwin' ] + } }, FlV4ckTxQ0a78hmBkzzc9A: { count: 405, @@ -165,6 +174,10 @@ describe('Get Beats Stats', () => { count: 0, names: [], }, + architecture: { + count: 0, + names: [] + } }, }); }); From 8597b4a741a8e9799d2fadd6386ffa8afea33ab1 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Fri, 17 Aug 2018 16:03:50 -0400 Subject: [PATCH 3/4] PR feedback --- .../monitoring/__tests__/get_beats_stats.js | 8 +++--- .../telemetry/monitoring/get_beats_stats.js | 26 +++++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js index d1ea9214b1642b..3abf1e73732390 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js @@ -14,7 +14,7 @@ const getBaseOptions = () => ({ clusterHostSets: {}, clusterInputSets: {}, clusterModuleSets: {}, - clusterArchitectureSets: {} + clusterArchitectureMaps: {} }); describe('Get Beats Stats', () => { @@ -114,7 +114,7 @@ describe('Get Beats Stats', () => { }, architecture: { count: 0, - names: [] + architectures: [] } }, }); @@ -147,7 +147,7 @@ describe('Get Beats Stats', () => { }, architecture: { count: 1, - names: [ 'x86_64/darwin' ] + architectures: [ 'x86_64/darwin' ] } }, FlV4ckTxQ0a78hmBkzzc9A: { @@ -176,7 +176,7 @@ describe('Get Beats Stats', () => { }, architecture: { count: 0, - names: [] + architectures: [] } }, }); diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js index 18bbda73196a69..17d733e97ffd01 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js @@ -28,7 +28,7 @@ const getBaseStats = () => ({ }, architecture: { count: 0, - names: [] + architectures: [] } }); @@ -41,7 +41,7 @@ const getBaseStats = () => ({ * @param {Object} clusterHostSets - the object keyed by cluster UUIDs to count the unique hosts * @param {Object} clusterModuleSets - the object keyed by cluster UUIDs to count the unique modules */ -export function processResults(results = [], { clusters, clusterHostSets, clusterInputSets, clusterModuleSets, clusterArchitectureSets }) { +export function processResults(results = [], { clusters, clusterHostSets, clusterInputSets, clusterModuleSets, clusterArchitectureMaps }) { const currHits = get(results, 'hits.hits', []); currHits.forEach(hit => { const clusterUuid = get(hit, '_source.cluster_uuid'); @@ -50,7 +50,7 @@ export function processResults(results = [], { clusters, clusterHostSets, cluste clusterHostSets[clusterUuid] = new Set(); clusterInputSets[clusterUuid] = new Set(); clusterModuleSets[clusterUuid] = new Set(); - clusterArchitectureSets[clusterUuid] = new Set(); + clusterArchitectureMaps[clusterUuid] = new Map(); } const processBeatsStatsResults = () => { @@ -105,11 +105,21 @@ export function processResults(results = [], { clusters, clusterHostSets, cluste const stateHost = get(hit, '_source.beats_state.state.host'); if (stateHost !== undefined) { - const hostSet = clusterArchitectureSets[clusterUuid]; - const hostArchPlatform = `${stateHost.architecture}/${stateHost.os.platform}`; - hostSet.add(hostArchPlatform); - clusters[clusterUuid].architecture.names = Array.from(hostSet); + const hostMap = clusterArchitectureMaps[clusterUuid]; + const hostKey = `${stateHost.architecture}/${stateHost.os.platform}`; + let os = hostMap.get(hostKey); + + if (!os) { // undefined if new + os = { name: stateHost.os.platform, architecture: stateHost.architecture, count: 0 }; + hostMap.set(hostKey, os); + } + + // total per os/arch + os.count += 1; + + // overall total (which should be the same number as the sum of all os.count values) clusters[clusterUuid].architecture.count += 1; + clusters[clusterUuid].architecture.architectures = Array.from(hostMap.keys()); } }; @@ -210,7 +220,7 @@ export async function getBeatsStats(server, callCluster, clusterUuids, start, en clusterHostSets: {}, // passed to processResults for tracking state in the results generation clusterInputSets: {}, // passed to processResults for tracking state in the results generation clusterModuleSets: {}, // passed to processResults for tracking state in the results generation - clusterArchitectureSets: {} // passed to processResults for tracking state in the results generation + clusterArchitectureMaps: {} // passed to processResults for tracking state in the results generation }; await Promise.all([ From 4b1c447438180e75a7938142815f50e1a669d1dc Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Mon, 20 Aug 2018 09:07:45 -0400 Subject: [PATCH 4/4] Use values instead of keys --- .../lib/telemetry/monitoring/__tests__/get_beats_stats.js | 8 +++++++- .../server/lib/telemetry/monitoring/get_beats_stats.js | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js index 3abf1e73732390..3cde267a1536bc 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_beats_stats.js @@ -147,7 +147,13 @@ describe('Get Beats Stats', () => { }, architecture: { count: 1, - architectures: [ 'x86_64/darwin' ] + architectures: [ + { + architecture: 'x86_64', + count: 1, + name: 'darwin' + } + ] } }, FlV4ckTxQ0a78hmBkzzc9A: { diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js index 17d733e97ffd01..ff0ddadac40f11 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_beats_stats.js @@ -119,7 +119,7 @@ export function processResults(results = [], { clusters, clusterHostSets, cluste // overall total (which should be the same number as the sum of all os.count values) clusters[clusterUuid].architecture.count += 1; - clusters[clusterUuid].architecture.architectures = Array.from(hostMap.keys()); + clusters[clusterUuid].architecture.architectures = Array.from(hostMap.values()); } };