Separate AWS module config file example into metricset specific (#12701)

* Separate AWS module config file example into metricset specific
elastic · Jul 10, 2019 · 0ae766b · 0ae766b
1 parent 3b00a72
commit 0ae766b
Show file tree

Hide file tree

Showing 12 changed files with 243 additions and 187 deletions.
diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc
@@ -50,55 +50,6 @@ The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]
 
-The aws.yml used for collecting metrics to display on the predefined aws
-dashboard is:
-[source,yaml]
-----
-- module: aws
-  period: 300s
-  metricsets:
-    - ec2
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-- module: aws
-  period: 300s
-  metricsets:
-    - sqs
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-
-- module: aws
-  period: 86400s
-  metricsets:
-    - s3_request
-    - s3_daily_storage
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-  # regions:
-  #    - us-west-1
-  #    - us-east-1
-- module: aws
-  period: 300s
-  metricsets:
-    - cloudwatch
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-  cloudwatch_metrics:
-    - namespace: AWS/EC2
-    - namespace: AWS/EBS
-    - namespace: AWS/Lambda
-    - namespace: AWS/ECS
-    - namespace: AWS/ELB
-----
-
 [float]
 == Metricsets
 
@@ -184,8 +135,8 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
-  #regions:
-  #  - us-west-1
+  regions:
+    - us-west-1
 - module: aws
   period: 86400s
   metricsets:
@@ -195,8 +146,6 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
-  #regions:
-  #  - us-west-1
 - module: aws
   period: 300s
   metricsets:
@@ -214,9 +163,6 @@ metricbeat.modules:
     - namespace: AWS/EBS
     - namespace: AWS/ELB
       tags.resource_type_filter: elasticloadbalancing
-  #regions:
-  #  - us-east-1
-  #  - us-east-2
 - module: aws
   period: 60s
   metricsets:

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -173,8 +173,8 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
-  #regions:
-  #  - us-west-1
+  regions:
+    - us-west-1
 - module: aws
   period: 86400s
   metricsets:
@@ -184,8 +184,6 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
-  #regions:
-  #  - us-west-1
 - module: aws
   period: 300s
   metricsets:
@@ -203,9 +201,6 @@ metricbeat.modules:
     - namespace: AWS/EBS
     - namespace: AWS/ELB
       tags.resource_type_filter: elasticloadbalancing
-  #regions:
-  #  - us-east-1
-  #  - us-east-2
 - module: aws
   period: 60s
   metricsets:

diff --git a/x-pack/metricbeat/module/aws/_meta/config.reference.yml b/x-pack/metricbeat/module/aws/_meta/config.reference.yml
@@ -0,0 +1,52 @@
+- module: aws
+  period: 300s
+  metricsets:
+    - ec2
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - sqs
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  regions:
+    - us-west-1
+- module: aws
+  period: 86400s
+  metricsets:
+    - s3_request
+    - s3_daily_storage
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EC2
+      metricname: CPUUtilization
+      dimensions:
+        - name: InstanceId
+          value: i-0686946e22cf9494a
+    - namespace: AWS/EBS
+    - namespace: AWS/ELB
+      tags.resource_type_filter: elasticloadbalancing
+- module: aws
+  period: 60s
+  metricsets:
+    - rds
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml
@@ -6,27 +6,8 @@
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
-- module: aws
-  period: 300s
-  metricsets:
-    - sqs
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-  #regions:
-  #  - us-west-1
-- module: aws
-  period: 86400s
-  metricsets:
-    - s3_request
-    - s3_daily_storage
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-  #regions:
-  #  - us-west-1
+  regions:
+    - us-east-1
 - module: aws
   period: 300s
   metricsets:
@@ -36,22 +17,6 @@
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
   cloudwatch_metrics:
-    - namespace: AWS/EC2
-      metricname: CPUUtilization
-      dimensions:
-        - name: InstanceId
-          value: i-0686946e22cf9494a
     - namespace: AWS/EBS
     - namespace: AWS/ELB
       tags.resource_type_filter: elasticloadbalancing
-  #regions:
-  #  - us-east-1
-  #  - us-east-2
-- module: aws
-  period: 60s
-  metricsets:
-    - rds
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc
@@ -43,55 +43,6 @@ The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]
 
-The aws.yml used for collecting metrics to display on the predefined aws
-dashboard is:
-[source,yaml]
-----
-- module: aws
-  period: 300s
-  metricsets:
-    - ec2
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-- module: aws
-  period: 300s
-  metricsets:
-    - sqs
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-
-- module: aws
-  period: 86400s
-  metricsets:
-    - s3_request
-    - s3_daily_storage
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-  # regions:
-  #    - us-west-1
-  #    - us-east-1
-- module: aws
-  period: 300s
-  metricsets:
-    - cloudwatch
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
-  cloudwatch_metrics:
-    - namespace: AWS/EC2
-    - namespace: AWS/EBS
-    - namespace: AWS/Lambda
-    - namespace: AWS/ECS
-    - namespace: AWS/ELB
-----
-
 [float]
 == Metricsets
 

diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc
@@ -13,7 +13,7 @@ tag:getResources
 ----
 
 [float]
-=== Module-specific configuration notes
+=== Metricset-specific configuration notes
 * *namespace*: The namespace to filter against. For example, AWS/EC2, AWS/S3.
 * *metricname*: The name of the metric to filter against. For example, CPUUtilization for EC2 instance.
 * *dimensions*: The dimensions to filter against. For example, InstanceId=i-123.
@@ -22,3 +22,109 @@ The format of each resource type is service[:resourceType].
 For example, specifying a resource type of ec2 returns all Amazon EC2 resources
 (which includes EC2 instances). Specifying a resource type of ec2:instance returns
 only EC2 instances.
+
+[float]
+=== Configuration example
+[source,yaml]
+----
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EBS <1>
+    - namespace: AWS/ELB <2>
+      tags.resource_type_filter: elasticloadbalancing
+    - namespace: AWS/EC2 <3>
+      metricname: CPUUtilization
+      dimensions:
+        - name: InstanceId
+          value: i-0686946e22cf9494a
+----
+
+<1> Users can configure the `cloudwatch` metricset to collect all metrics from one
+specific namespace, such as `AWS/EBS`.
+
+<2> `cloudwatch` metricset also has the ability to collect tags from AWS resources.
+If user specify `tags.resource_type_filter`, then tags will be collected and stored
+as a part of the event. Please see https://docs.aws.amazon.com/resourcegroupstagging/latest/APIReference/API_GetResources.html[AWS API GetResources]
+for more details about `tags.resource_type_filter`.
+
+<3> If users knows exactly what are the cloudwatch metrics they want to collect,
+this configuration format can be used. `namespace` and `metricname` need to be
+specified and `dimensions` can be used to filter cloudwatch metrics. Please see
+https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/list-metrics.html[AWS List Metrics]
+for more details.
+
+[float]
+=== More examples
+With the configuration below, users will be able to collect cloudwatch metrics
+from EBS, ELB and EC2 without tag information.
+
+[source,yaml]
+----
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EBS
+    - namespace: AWS/ELB
+    - namespace: AWS/EC2
+----
+
+With the configuration below, users will be able to collect cloudwatch metrics
+from EBS, ELB and EC2 with tags from these services.
+
+[source,yaml]
+----
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EBS
+      tags.resource_type_filter: ebs
+    - namespace: AWS/ELB
+      tags.resource_type_filter: elasticloadbalancing
+    - namespace: AWS/EC2
+      tags.resource_type_filter: ec2:instance
+----
+
+With the configuration below, users will be able to collect specific cloudwatch
+metrics. For example CPUUtilization metric from EC2 instance i-123 and NetworkIn
+metric from EC2 instance i-456.
+[source,yaml]
+----
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EC2
+      metricname: CPUUtilization
+      dimensions:
+        - name: InstanceId
+          value: i-123
+    - namespace: AWS/EC2
+      metricname: NetworkIn
+      dimensions:
+        - name: InstanceId
+          value: i-456
+----