Add statistic option to cloudwatch metricset config (#12840)

* Add statistic option to cloudwatch metricset config
elastic · Jul 25, 2019 · 91c0a08 · 91c0a08
1 parent b74e19c
commit 91c0a08
Show file tree

Hide file tree

Showing 11 changed files with 748 additions and 197 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -40,6 +40,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add new option `OpMultiplyBuckets` to scale histogram buckets to avoid decimal points in final events {pull}10994[10994]
 - system/raid metricset now uses /sys/block instead of /proc/mdstat for data. {pull}11613[11613]
 - kubernetes.container.cpu.limit.cores and kubernetes.container.cpu.requests.cores are now floats. {issue}11975[11975]
+- Add statistic option into cloudwatch metricset. If there is no statistic method specified, default is to collect Average, Sum, Maximum, Minimum and SampleCount. {issue}12370[12370] {pull}12840[12840]
 
 *Packetbeat*
 
@@ -292,6 +293,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Oracle Tablespaces Dashboard {pull}12736[12736]
 - Collect client provided name for rabbitmq connection. {issue}12851[12851] {pull}12852[12852]
 - Add support to load default aws config file to get credentials. {pull}12727[12727] {issue}12708[12708]
+- Add statistic option into cloudwatch metricset. {issue}12370[12370] {pull}12840[12840]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -860,7 +860,7 @@ type: keyword
 
 --
 
-*`aws.cloudwatch.metrics.*`*::
+*`aws.cloudwatch.metrics.*.*`*::
 +
 --
 Metrics that returned from Cloudwatch api query.

diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml
@@ -8,7 +8,11 @@
   period: 300s
   metricsets:
     - cloudwatch
-  cloudwatch_metrics:
-    - namespace: AWS/EBS
-    - namespace: AWS/ELB
-      tags.resource_type_filter: elasticloadbalancing
+  metrics:
+    - namespace: AWS/EC2
+      name: ["CPUUtilization", "DiskWriteOps"]
+      tags.resource_type_filter: ec2:intance
+      #dimensions:
+      #  - name: InstanceId
+      #    value: i-0686946e22cf9494a
+      statistic: ["Average", "Maximum"]
diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json
@@ -3,47 +3,239 @@
     "aws": {
         "cloudwatch": {
             "dimensions": {
-                "DBInstanceIdentifier": "test1"
+                "DBClusterIdentifier": "test1-cluster",
+                "Role": "WRITER"
             },
             "metrics": {
-                "ActiveTransactions": 0,
-                "AuroraBinlogReplicaLag": 0,
-                "AuroraReplicaLagMaximum": 18.62599983215332,
-                "AuroraReplicaLagMinimum": 18.62599983215332,
-                "BinLogDiskUsage": 0,
-                "BlockedTransactions": 0,
-                "BufferCacheHitRatio": 100,
-                "CPUUtilization": 3.2,
-                "CommitLatency": 7.324593333333333,
-                "CommitThroughput": 0.49995937707131555,
-                "DDLLatency": 0,
-                "DDLThroughput": 0,
-                "DMLLatency": 0.15980666666666668,
-                "DMLThroughput": 0.49995937707131555,
-                "DatabaseConnections": 0,
-                "Deadlocks": 0,
-                "DeleteLatency": 0,
-                "DeleteThroughput": 0,
-                "EngineUptime": 5698620,
-                "FreeLocalStorage": 30215283507.2,
-                "FreeableMemory": 4574632345.6,
-                "InsertLatency": 0.15980666666666668,
-                "InsertThroughput": 0.49995937707131555,
-                "LoginFailures": 0,
-                "NetworkReceiveThroughput": 0.6999953344221976,
-                "NetworkThroughput": 1.3999906688443953,
-                "NetworkTransmitThroughput": 0.6999953344221976,
-                "Queries": 8.796891844051007,
-                "ResultSetCacheHitRatio": 0,
-                "SelectLatency": 0.17348499249549354,
-                "SelectThroughput": 3.0864647634387916,
-                "UpdateLatency": 0,
-                "UpdateThroughput": 0
+                "ActiveTransactions": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "AuroraBinlogReplicaLag": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "AuroraReplicaLagMaximum": {
+                    "avg": 20.067200088500975,
+                    "count": 5,
+                    "max": 26.357999801635742,
+                    "min": 13.218999862670898,
+                    "sum": 100.33600044250488
+                },
+                "AuroraReplicaLagMinimum": {
+                    "avg": 20.067200088500975,
+                    "count": 5,
+                    "max": 26.357999801635742,
+                    "min": 13.218999862670898,
+                    "sum": 100.33600044250488
+                },
+                "BinLogDiskUsage": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "BlockedTransactions": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "BufferCacheHitRatio": {
+                    "avg": 100,
+                    "max": 100,
+                    "min": 100,
+                    "sum": 500
+                },
+                "CPUUtilization": {
+                    "avg": 3.8,
+                    "max": 5,
+                    "min": 3,
+                    "sum": 19
+                },
+                "CommitLatency": {
+                    "avg": 7.25688,
+                    "count": 5,
+                    "max": 8.186233333333332,
+                    "min": 6.463433333333333,
+                    "sum": 36.2844
+                },
+                "CommitThroughput": {
+                    "avg": 0.49994837279396603,
+                    "count": 5,
+                    "max": 0.5001000200040008,
+                    "min": 0.4998000799680128,
+                    "sum": 2.49974186396983
+                },
+                "DDLLatency": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "DDLThroughput": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "DMLLatency": {
+                    "avg": 0.16545333333333337,
+                    "count": 5,
+                    "max": 0.17093333333333333,
+                    "min": 0.16276666666666667,
+                    "sum": 0.8272666666666668
+                },
+                "DMLThroughput": {
+                    "avg": 0.49994837279396603,
+                    "count": 5,
+                    "max": 0.5001000200040008,
+                    "min": 0.4998000799680128,
+                    "sum": 2.49974186396983
+                },
+                "DatabaseConnections": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "Deadlocks": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "DeleteLatency": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "DeleteThroughput": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "EngineUptime": {
+                    "avg": 1244563,
+                    "count": 5,
+                    "max": 1244683,
+                    "min": 1244443,
+                    "sum": 6222815
+                },
+                "FreeLocalStorage": {
+                    "avg": 31932801024,
+                    "count": 5,
+                    "max": 31932895232,
+                    "min": 31932706816,
+                    "sum": 159664005120
+                },
+                "FreeableMemory": {
+                    "avg": 4474304102.4,
+                    "max": 4476133376,
+                    "min": 4473532416,
+                    "sum": 22371520512
+                },
+                "InsertLatency": {
+                    "avg": 0.16545333333333337,
+                    "count": 5,
+                    "max": 0.17093333333333333,
+                    "min": 0.16276666666666667,
+                    "sum": 0.8272666666666668
+                },
+                "InsertThroughput": {
+                    "avg": 0.49994837279396603,
+                    "count": 5,
+                    "max": 0.5001000200040008,
+                    "min": 0.4998000799680128,
+                    "sum": 2.49974186396983
+                },
+                "LoginFailures": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "NetworkReceiveThroughput": {
+                    "avg": 0.8400256730059361,
+                    "max": 1.4001633523911123,
+                    "min": 0.6999300069993001,
+                    "sum": 4.200128365029681
+                },
+                "NetworkThroughput": {
+                    "avg": 1.6800513460118722,
+                    "count": 5,
+                    "max": 2.8003267047822247,
+                    "min": 1.3998600139986002,
+                    "sum": 8.400256730059361
+                },
+                "NetworkTransmitThroughput": {
+                    "avg": 0.8400256730059361,
+                    "count": 5,
+                    "max": 1.4001633523911123,
+                    "min": 0.6999300069993001,
+                    "sum": 4.200128365029681
+                },
+                "Queries": {
+                    "avg": 9.018293376391766,
+                    "count": 5,
+                    "max": 9.842454118509144,
+                    "min": 8.654615795090715,
+                    "sum": 45.09146688195883
+                },
+                "ResultSetCacheHitRatio": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "SelectLatency": {
+                    "avg": 0.18611627194800234,
+                    "count": 5,
+                    "max": 0.20127322404371584,
+                    "min": 0.1782340425531915,
+                    "sum": 0.9305813597400117
+                },
+                "SelectThroughput": {
+                    "avg": 3.1130005268141185,
+                    "count": 5,
+                    "max": 3.2153805144608825,
+                    "min": 3.0339401213576047,
+                    "sum": 15.565002634070593
+                },
+                "UpdateLatency": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                },
+                "UpdateThroughput": {
+                    "avg": 0,
+                    "count": 5,
+                    "max": 0,
+                    "min": 0,
+                    "sum": 0
+                }
             },
             "namespace": "AWS/RDS"
-        },
-        "tags": {
-            "workload-type": "other"
         }
     },
     "cloud": {

diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc
@@ -72,7 +72,7 @@ from EBS, ELB and EC2 without tag information.
   period: 300s
   metricsets:
     - cloudwatch
-  cloudwatch_metrics:
+  metrics:
     - namespace: AWS/EBS
     - namespace: AWS/ELB
     - namespace: AWS/EC2
@@ -87,7 +87,7 @@ from EBS, ELB and EC2 with tags from these services.
   period: 300s
   metricsets:
     - cloudwatch
-  cloudwatch_metrics:
+  metrics:
     - namespace: AWS/EBS
       tags.resource_type_filter: ebs
     - namespace: AWS/ELB
@@ -97,23 +97,26 @@ from EBS, ELB and EC2 with tags from these services.
 ----
 
 With the configuration below, users will be able to collect specific cloudwatch
-metrics. For example CPUUtilization metric from EC2 instance i-123 and NetworkIn
-metric from EC2 instance i-456.
+metrics. For example CPUUtilization metric(average) from EC2 instance i-123 and NetworkIn
+metric(average) from EC2 instance i-456.
 [source,yaml]
 ----
 - module: aws
   period: 300s
   metricsets:
     - cloudwatch
-  cloudwatch_metrics:
+  metrics:
     - namespace: AWS/EC2
-      metricname: CPUUtilization
+      name: ["CPUUtilization"]
+      tags.resource_type_filter: ec2:intance
       dimensions:
         - name: InstanceId
           value: i-123
+      statistic: ["Average"]
     - namespace: AWS/EC2
-      metricname: NetworkIn
+      name: ["NetworkIn"]
       dimensions:
         - name: InstanceId
           value: i-456
+      statistic: ["Average"]
 ----
diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml
@@ -8,7 +8,7 @@
       type: keyword
       description: >
         The namespace specified when query cloudwatch api.
-    - name: metrics.*
+    - name: metrics.*.*
       type: object
       object_type: double
       object_type_mapping_type: "*"