From b7e211c17db83dd5689a09b69341b9bb1fa628c5 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Wed, 14 Aug 2019 18:39:49 +0200 Subject: [PATCH 1/4] Use sum for aggregations. --- .../lib/adapters/metrics/adapter_types.ts | 1 + .../metrics/models/aws/aws_diskio_bytes.ts | 22 ++++++++++--------- .../metrics/models/aws/aws_diskio_ops.ts | 22 +++++++++++++------ .../metrics/models/aws/aws_network_bytes.ts | 22 ++++++++++--------- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts index 8b975feb7a0fa9..3fd3cef4e59c9f 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts @@ -47,6 +47,7 @@ export enum InfraMetricModelMetricType { positive_only = 'positive_only', // eslint-disable-line @typescript-eslint/camelcase derivative = 'derivative', count = 'count', + sum = 'sum', } export interface InfraMetricModel { diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts index 90146e6de5370d..a3bf53255d8618 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts @@ -11,6 +11,8 @@ import { } from '../../adapter_types'; import { InfraMetric } from '../../../../../graphql/types'; +// see discussion in: https://github.com/elastic/kibana/issues/42687 + export const awsDiskioBytes: InfraMetricModelCreator = ( timeField, indexPattern, @@ -30,14 +32,14 @@ export const awsDiskioBytes: InfraMetricModelCreator = ( metrics: [ { field: 'aws.ec2.diskio.write.bytes', - id: 'max-diskio-out', - type: InfraMetricModelMetricType.max, + id: 'sum-diskio-out', + type: InfraMetricModelMetricType.sum, }, { - id: 'by-second-max-diskio-out', + id: 'by-second-sum-diskio-out', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-max', name: 'max', field: 'max-diskio-out' }], - script: 'params.max / 300', // TODO: https://github.com/elastic/kibana/issues/42687 + variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-out' }], + script: 'params.sum / 300', }, ], split_mode: 'everything', @@ -47,14 +49,14 @@ export const awsDiskioBytes: InfraMetricModelCreator = ( metrics: [ { field: 'aws.ec2.diskio.read.bytes', - id: 'max-diskio-in', - type: InfraMetricModelMetricType.max, + id: 'sum-diskio-in', + type: InfraMetricModelMetricType.sum, }, { - id: 'inverted-by-second-max-diskio-in', + id: 'inverted-by-second-sum-diskio-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-max', name: 'max', field: 'max-diskio-in' }], - script: 'params.max / -300', // TODO: https://github.com/elastic/kibana/issues/42687 + variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-in' }], + script: 'params.sum / -300', }, ], split_mode: 'everything', diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts index 50394488f5343a..93daa24a05a086 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts @@ -11,6 +11,8 @@ import { } from '../../adapter_types'; import { InfraMetric } from '../../../../../graphql/types'; +// see discussion in: https://github.com/elastic/kibana/issues/42687 + export const awsDiskioOps: InfraMetricModelCreator = ( timeField, indexPattern, @@ -30,8 +32,14 @@ export const awsDiskioOps: InfraMetricModelCreator = ( metrics: [ { field: 'aws.ec2.diskio.write.count', - id: 'max-diskio-writes', - type: InfraMetricModelMetricType.max, + id: 'sum-diskio-writes', + type: InfraMetricModelMetricType.sum, + }, + { + id: 'by-second-sum-diskio-writes', + type: InfraMetricModelMetricType.calculation, + variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-writes' }], + script: 'params.sum / 300', }, ], split_mode: 'everything', @@ -41,14 +49,14 @@ export const awsDiskioOps: InfraMetricModelCreator = ( metrics: [ { field: 'aws.ec2.diskio.read.count', - id: 'max-diskio-reads', - type: InfraMetricModelMetricType.max, + id: 'sum-diskio-reads', + type: InfraMetricModelMetricType.sum, }, { - id: 'inverted-avg-diskio-reads', + id: 'inverted-by-second-sum-diskio-reads', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-max', name: 'max', field: 'max-diskio-reads' }], - script: 'params.max * -1', + variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-reads' }], + script: 'params.sum / -300', }, ], split_mode: 'everything', diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts index 0b62b4d5f29b53..104480a396f3a2 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts @@ -11,6 +11,8 @@ import { } from '../../adapter_types'; import { InfraMetric } from '../../../../../graphql/types'; +// see discussion in: https://github.com/elastic/kibana/issues/42687 + export const awsNetworkBytes: InfraMetricModelCreator = ( timeField, indexPattern, @@ -30,14 +32,14 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( metrics: [ { field: 'aws.ec2.network.out.bytes', - id: 'max-net-out', - type: InfraMetricModelMetricType.max, + id: 'sum-net-out', + type: InfraMetricModelMetricType.sum, }, { - id: 'by-second-max-net-out', + id: 'by-second-sum-net-out', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-max', name: 'max', field: 'max-net-out' }], - script: 'params.max / 300', // TODO: https://github.com/elastic/kibana/issues/42687 + variables: [{ id: 'var-sum', name: 'sum', field: 'sum-net-out' }], + script: 'params.sum / 300', }, ], split_mode: 'everything', @@ -47,14 +49,14 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( metrics: [ { field: 'aws.ec2.network.in.bytes', - id: 'max-net-in', - type: InfraMetricModelMetricType.max, + id: 'sum-net-in', + type: InfraMetricModelMetricType.sum, }, { - id: 'inverted-by-second-max-net-in', + id: 'inverted-by-second-sum-net-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-max', name: 'max', field: 'max-net-in' }], - script: 'params.max / -300', // TODO: https://github.com/elastic/kibana/issues/42687 + variables: [{ id: 'var-sum', name: 'sum', field: 'sum-net-in' }], + script: 'params.sum / -300', }, ], split_mode: 'everything', From 8fb96275a0c6f3385c43f28639d6b4e76714d653 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 16 Aug 2019 13:49:03 +0200 Subject: [PATCH 2/4] Use cumulative sum/derivative instead of hard coded rate --- .../lib/adapters/metrics/adapter_types.ts | 1 + .../metrics/models/aws/aws_diskio_bytes.ts | 30 ++++++++++++++----- .../metrics/models/aws/aws_diskio_ops.ts | 30 ++++++++++++++----- .../metrics/models/aws/aws_network_bytes.ts | 30 ++++++++++++++----- 4 files changed, 70 insertions(+), 21 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts index 3fd3cef4e59c9f..adb8c811ed57d9 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/adapter_types.ts @@ -48,6 +48,7 @@ export enum InfraMetricModelMetricType { derivative = 'derivative', count = 'count', sum = 'sum', + cumulative_sum = 'cumulative_sum', // eslint-disable-line @typescript-eslint/camelcase } export interface InfraMetricModel { diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts index a3bf53255d8618..471e219fa7cfb4 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts @@ -36,10 +36,15 @@ export const awsDiskioBytes: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.sum, }, { - id: 'by-second-sum-diskio-out', - type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-out' }], - script: 'params.sum / 300', + id: 'csum-sum-diskio-out', + field: 'sum-diskio-out', + type: InfraMetricModelMetricType.cumulative_sum, + }, + { + id: 'deriv-csum-sum-diskio-out', + unit: '1s', + type: InfraMetricModelMetricType.derivative, + field: 'csum-sum-diskio-out', }, ], split_mode: 'everything', @@ -53,10 +58,21 @@ export const awsDiskioBytes: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.sum, }, { - id: 'inverted-by-second-sum-diskio-in', + id: 'csum-sum-diskio-in', + field: 'sum-diskio-in', + type: InfraMetricModelMetricType.cumulative_sum, + }, + { + id: 'deriv-csum-sum-diskio-in', + unit: '1s', + type: InfraMetricModelMetricType.derivative, + field: 'csum-sum-diskio-in', + }, + { + id: 'inverted-deriv-csum-sum-diskio-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-in' }], - script: 'params.sum / -300', + variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-csum-sum-diskio-in' }], + script: 'params.rate * -1', }, ], split_mode: 'everything', diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts index 93daa24a05a086..942e43a5e60cba 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts @@ -36,10 +36,15 @@ export const awsDiskioOps: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.sum, }, { - id: 'by-second-sum-diskio-writes', - type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-writes' }], - script: 'params.sum / 300', + id: 'csum-sum-diskio-writes', + field: 'sum-diskio-writes', + type: InfraMetricModelMetricType.cumulative_sum, + }, + { + id: 'deriv-csum-sum-diskio-writes', + unit: '1s', + type: InfraMetricModelMetricType.derivative, + field: 'csum-sum-diskio-writes', }, ], split_mode: 'everything', @@ -53,10 +58,21 @@ export const awsDiskioOps: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.sum, }, { - id: 'inverted-by-second-sum-diskio-reads', + id: 'csum-sum-diskio-reads', + field: 'sum-diskio-reads', + type: InfraMetricModelMetricType.cumulative_sum, + }, + { + id: 'deriv-csum-sum-diskio-reads', + unit: '1s', + type: InfraMetricModelMetricType.derivative, + field: 'csum-sum-diskio-reads', + }, + { + id: 'inverted-deriv-csum-sum-diskio-reads', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-sum', name: 'sum', field: 'sum-diskio-reads' }], - script: 'params.sum / -300', + variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-csum-sum-diskio-reads' }], + script: 'params.rate * -1', }, ], split_mode: 'everything', diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts index 104480a396f3a2..b18a20c40d8771 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts @@ -36,10 +36,15 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.sum, }, { - id: 'by-second-sum-net-out', - type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-sum', name: 'sum', field: 'sum-net-out' }], - script: 'params.sum / 300', + id: 'csum-sum-net-out', + field: 'sum-net-out', + type: InfraMetricModelMetricType.cumulative_sum, + }, + { + id: 'deriv-csum-sum-net-out', + unit: '1s', + type: InfraMetricModelMetricType.derivative, + field: 'csum-sum-net-out', }, ], split_mode: 'everything', @@ -53,10 +58,21 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.sum, }, { - id: 'inverted-by-second-sum-net-in', + id: 'csum-sum-net-in', + field: 'sum-net-in', + type: InfraMetricModelMetricType.cumulative_sum, + }, + { + id: 'deriv-csum-sum-net-in', + unit: '1s', + type: InfraMetricModelMetricType.derivative, + field: 'csum-sum-net-in', + }, + { + id: 'inverted-deriv-csum-sum-net-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-sum', name: 'sum', field: 'sum-net-in' }], - script: 'params.sum / -300', + variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-cumsum-sum-net-in' }], + script: 'params.rate * -1', }, ], split_mode: 'everything', From 479d5bb9caf9b5ac51fc4ff700547fb4fcaafe59 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 16 Aug 2019 17:48:24 +0200 Subject: [PATCH 3/4] Fix typo. --- .../server/lib/adapters/metrics/models/aws/aws_network_bytes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts index b18a20c40d8771..aa250d71effb35 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts @@ -71,7 +71,7 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( { id: 'inverted-deriv-csum-sum-net-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-cumsum-sum-net-in' }], + variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-csum-sum-net-in' }], script: 'params.rate * -1', }, ], From 867c5c27b7c14ee30ae41275818155ba0f79c389 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 16 Aug 2019 17:55:30 +0200 Subject: [PATCH 4/4] Add positive_only after derivatives. --- .../metrics/models/aws/aws_diskio_bytes.ts | 14 ++++++++++++-- .../metrics/models/aws/aws_diskio_ops.ts | 16 ++++++++++++++-- .../metrics/models/aws/aws_network_bytes.ts | 14 ++++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts index 471e219fa7cfb4..15077c3c64cf7d 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_bytes.ts @@ -46,6 +46,11 @@ export const awsDiskioBytes: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.derivative, field: 'csum-sum-diskio-out', }, + { + id: 'posonly-deriv-csum-sum-diskio-out', + field: 'deriv-csum-sum-diskio-out', + type: InfraMetricModelMetricType.positive_only, + }, ], split_mode: 'everything', }, @@ -69,9 +74,14 @@ export const awsDiskioBytes: InfraMetricModelCreator = ( field: 'csum-sum-diskio-in', }, { - id: 'inverted-deriv-csum-sum-diskio-in', + id: 'posonly-deriv-csum-sum-diskio-in', + field: 'deriv-csum-sum-diskio-in', + type: InfraMetricModelMetricType.positive_only, + }, + { + id: 'inverted-posonly-deriv-csum-sum-diskio-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-csum-sum-diskio-in' }], + variables: [{ id: 'var-rate', name: 'rate', field: 'posonly-deriv-csum-sum-diskio-in' }], script: 'params.rate * -1', }, ], diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts index 942e43a5e60cba..3df6124d22a7fb 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_diskio_ops.ts @@ -46,6 +46,11 @@ export const awsDiskioOps: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.derivative, field: 'csum-sum-diskio-writes', }, + { + id: 'posonly-deriv-csum-sum-diskio-writes', + field: 'deriv-csum-sum-diskio-writes', + type: InfraMetricModelMetricType.positive_only, + }, ], split_mode: 'everything', }, @@ -69,9 +74,16 @@ export const awsDiskioOps: InfraMetricModelCreator = ( field: 'csum-sum-diskio-reads', }, { - id: 'inverted-deriv-csum-sum-diskio-reads', + id: 'posonly-deriv-csum-sum-diskio-reads', + field: 'deriv-csum-sum-diskio-reads', + type: InfraMetricModelMetricType.positive_only, + }, + { + id: 'inverted-posonly-deriv-csum-sum-diskio-reads', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-csum-sum-diskio-reads' }], + variables: [ + { id: 'var-rate', name: 'rate', field: 'posonly-deriv-csum-sum-diskio-reads' }, + ], script: 'params.rate * -1', }, ], diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts index aa250d71effb35..3c9842c4ada8f8 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/metrics/models/aws/aws_network_bytes.ts @@ -46,6 +46,11 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( type: InfraMetricModelMetricType.derivative, field: 'csum-sum-net-out', }, + { + id: 'posonly-deriv-csum-sum-net-out', + field: 'deriv-csum-sum-net-out', + type: InfraMetricModelMetricType.positive_only, + }, ], split_mode: 'everything', }, @@ -69,9 +74,14 @@ export const awsNetworkBytes: InfraMetricModelCreator = ( field: 'csum-sum-net-in', }, { - id: 'inverted-deriv-csum-sum-net-in', + id: 'posonly-deriv-csum-sum-net-in', + field: 'deriv-csum-sum-net-in', + type: InfraMetricModelMetricType.positive_only, + }, + { + id: 'inverted-posonly-deriv-csum-sum-net-in', type: InfraMetricModelMetricType.calculation, - variables: [{ id: 'var-rate', name: 'rate', field: 'deriv-csum-sum-net-in' }], + variables: [{ id: 'var-rate', name: 'rate', field: 'posonly-deriv-csum-sum-net-in' }], script: 'params.rate * -1', }, ],