From a44da4cd69d53ae527e5a7e119d694e32f9a707e Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:01:21 +0100 Subject: [PATCH 01/18] Update dependency on elastic-package --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 75ba146eecc..ac3caea17c7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/blang/semver v3.5.1+incompatible - github.com/elastic/elastic-package v0.0.0-20210225101737-c8b00a93596e + github.com/elastic/elastic-package v0.0.0-20210301151806-9253ee3cdcfe github.com/elastic/package-registry v0.17.0 github.com/magefile/mage v1.11.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index e1f9b982e40..6870913fe97 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/elastic/elastic-package v0.0.0-20210225101737-c8b00a93596e h1:robuyReHsYgD3qlg9Z7Ro7Xqbzm0XqS03V/XKkuriOg= -github.com/elastic/elastic-package v0.0.0-20210225101737-c8b00a93596e/go.mod h1:hzJTWwSTpP3mLK9NcjnoLifXmOjitGbVwgI/RPmsEGE= +github.com/elastic/elastic-package v0.0.0-20210301151806-9253ee3cdcfe h1:zNv+fOiAXa2i9aqzxw+ht+ftfntSXLNJUlSd9oPUjwQ= +github.com/elastic/elastic-package v0.0.0-20210301151806-9253ee3cdcfe/go.mod h1:x4ojhiQp1YW2dADwv8QYyx9hNLuto4jarprEO8qUx7c= github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M= github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= @@ -95,8 +95,8 @@ github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6 h1:Ehbr7du4rSSEy github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= github.com/elastic/package-registry v0.17.0 h1:Gh7u3TlHA3GJh+C/OZ8Pf4EUrFxcCXMAe2kUCjAiYgQ= github.com/elastic/package-registry v0.17.0/go.mod h1:fMVt9ozLSPAIgYTDgV23IZrSoDKZma7VKpA4uSkfPts= -github.com/elastic/package-spec/code/go v0.0.0-20210210152225-3f48d5aaa17e h1:kL1ypDLbxsEyPxkjmvccMau5Ap5s3yMk4qDO3xlbEos= -github.com/elastic/package-spec/code/go v0.0.0-20210210152225-3f48d5aaa17e/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= +github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3 h1:SX1mNX3H7+NQ+88eBu8nsWJGCLUQnbGdxh/qpS/6Wq8= +github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= From 779528bb2c6d03460edbb56ea6175f6d4d755a6f Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:13:11 +0100 Subject: [PATCH 02/18] Adjust AWS integration --- packages/aws/changelog.yml | 5 + .../aws/data_stream/billing/fields/fields.yml | 6 + .../aws/data_stream/billing/sample_event.json | 97 +- .../cloudwatch_metrics/sample_event.json | 109 +- .../data_stream/dynamodb/fields/fields.yml | 6 + .../data_stream/dynamodb/sample_event.json | 119 +- .../aws/data_stream/ebs/fields/fields.yml | 6 + .../aws/data_stream/ebs/sample_event.json | 136 +- .../data_stream/ec2_metrics/sample_event.json | 220 +- .../data_stream/elb_metrics/fields/fields.yml | 6 + .../data_stream/elb_metrics/sample_event.json | 130 +- .../aws/data_stream/lambda/fields/fields.yml | 6 + .../aws/data_stream/lambda/sample_event.json | 117 +- .../data_stream/natgateway/fields/fields.yml | 6 + .../data_stream/natgateway/sample_event.json | 172 +- .../aws/data_stream/rds/sample_event.json | 188 +- .../s3_daily_storage/sample_event.json | 102 +- .../data_stream/s3_request/sample_event.json | 128 +- .../aws/data_stream/sns/fields/fields.yml | 6 + .../aws/data_stream/sns/sample_event.json | 116 +- .../aws/data_stream/sqs/sample_event.json | 116 +- .../transitgateway/fields/fields.yml | 6 + .../transitgateway/sample_event.json | 130 +- .../aws/data_stream/usage/fields/fields.yml | 6 + .../aws/data_stream/usage/sample_event.json | 106 +- .../aws/data_stream/vpn/fields/fields.yml | 6 + .../aws/data_stream/vpn/sample_event.json | 108 +- packages/aws/docs/README.md | 2112 +++++++---------- packages/aws/manifest.yml | 2 +- 29 files changed, 1810 insertions(+), 2463 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 26cce41a30b..aa7e0bb62a3 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.4.1" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.4.0" changes: - description: Add changes to use ECS 1.8 fields. diff --git a/packages/aws/data_stream/billing/fields/fields.yml b/packages/aws/data_stream/billing/fields/fields.yml index 58e447476f7..5b3ee582a6e 100644 --- a/packages/aws/data_stream/billing/fields/fields.yml +++ b/packages/aws/data_stream/billing/fields/fields.yml @@ -78,3 +78,9 @@ object_type: keyword object_type_mapping_type: "*" description: Cost explorer group by key values. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/billing/sample_event.json b/packages/aws/data_stream/billing/sample_event.json index 9c14baf14b4..0a252492f01 100644 --- a/packages/aws/data_stream/billing/sample_event.json +++ b/packages/aws/data_stream/billing/sample_event.json @@ -1,67 +1,48 @@ { - "_index": "metrics-aws.billing-default-000001", - "_id": "IMxJXHIBpGMSUzkZo-s0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } + "@timestamp": "2020-05-28T17:17:06.212Z", + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.billing", + "module": "aws", + "duration": 1938760247 + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "billing": { + "metrics": { + "EstimatedCharges": { + "max": 1625.41 } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" } }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.billing", - "namespace": "default" + "cloudwatch": { + "namespace": "AWS/Billing" }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + "dimensions": { + "Currency": "USD" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:06.212Z" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590686226212 - ] + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/cloudwatch_metrics/sample_event.json b/packages/aws/data_stream/cloudwatch_metrics/sample_event.json index b79ac10d015..431705cacd8 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/sample_event.json +++ b/packages/aws/data_stream/cloudwatch_metrics/sample_event.json @@ -1,72 +1,53 @@ { - "_index": "metrics-aws.cloudwatch_metrics-default-000001", - "_id": "-sxJXHIBpGMSUzkZxex8", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "stream": { - "dataset": "aws.cloudwatch_metrics", - "namespace": "default", - "type": "metrics" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } + "@timestamp": "2020-05-28T17:17:02.812Z", + "event": { + "duration": 14119105951, + "dataset": "aws.cloudwatch", + "module": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "dimensions": { + "InstanceId": "i-0830bfecfa7173cbe" }, - "aws": { - "dimensions": { - "InstanceId": "i-0830bfecfa7173cbe" - }, - "ec2": { - "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, - "CPUUtilization": { - "avg": 0.7661943132361363, - "max": 0.833333333333333 - } + "ec2": { + "metrics": { + "DiskWriteOps": { + "avg": 0, + "max": 0 + }, + "CPUUtilization": { + "avg": 0.7661943132361363, + "max": 0.833333333333333 } - }, - "cloudwatch": { - "namespace": "AWS/EC2" } }, - "metricset": { - "period": 300000, - "name": "cloudwatch" + "cloudwatch": { + "namespace": "AWS/EC2" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:02.812Z" - ] - }, - "sort": [ - 1590686222812 - ] + "metricset": { + "period": 300000, + "name": "cloudwatch" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/dynamodb/fields/fields.yml b/packages/aws/data_stream/dynamodb/fields/fields.yml index 892fc7edee9..abd232950d2 100644 --- a/packages/aws/data_stream/dynamodb/fields/fields.yml +++ b/packages/aws/data_stream/dynamodb/fields/fields.yml @@ -107,3 +107,9 @@ type: double description: | The percentage of provisioned write capacity utilized by the highest provisioned write table or global secondary index of an account. +- name: aws.cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/dynamodb/sample_event.json b/packages/aws/data_stream/dynamodb/sample_event.json index 8866a6ac1c5..6973aa2c906 100644 --- a/packages/aws/data_stream/dynamodb/sample_event.json +++ b/packages/aws/data_stream/dynamodb/sample_event.json @@ -1,78 +1,59 @@ { - "_index": "metrics-aws.dynamodb-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.dynamodb", - "namespace": "default" - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "dimensions": { + "TableName": "TryDaxTable3" }, - "aws": { - "dimensions": { - "TableName": "TryDaxTable3" - }, - "dynamodb": { - "metrics": { - "ProvisionedWriteCapacityUnits": { - "avg": 1 - }, - "ProvisionedReadCapacityUnits": { - "avg": 1 - }, - "ConsumedWriteCapacityUnits": { - "avg": 0, - "sum": 0 - }, - "ConsumedReadCapacityUnits": { - "avg": 0, - "sum": 0 - } + "dynamodb": { + "metrics": { + "ProvisionedWriteCapacityUnits": { + "avg": 1 + }, + "ProvisionedReadCapacityUnits": { + "avg": 1 + }, + "ConsumedWriteCapacityUnits": { + "avg": 0, + "sum": 0 + }, + "ConsumedReadCapacityUnits": { + "avg": 0, + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/DynamoDB" } }, - "metricset": { - "name": "dynamodb", - "period": 300000 + "cloudwatch": { + "namespace": "AWS/DynamoDB" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } \ No newline at end of file diff --git a/packages/aws/data_stream/ebs/fields/fields.yml b/packages/aws/data_stream/ebs/fields/fields.yml index 371f83b07b0..c230284e0da 100644 --- a/packages/aws/data_stream/ebs/fields/fields.yml +++ b/packages/aws/data_stream/ebs/fields/fields.yml @@ -46,3 +46,9 @@ - name: VolumeIdleTime.sum type: double description: The total number of seconds in a specified period of time when no read or write operations were submitted. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/ebs/sample_event.json b/packages/aws/data_stream/ebs/sample_event.json index e0daba4d006..ce81b383a50 100644 --- a/packages/aws/data_stream/ebs/sample_event.json +++ b/packages/aws/data_stream/ebs/sample_event.json @@ -1,90 +1,66 @@ { - "_index": "metrics-aws.ebs-default-000001", - "_id": "_89uXHIBpGMSUzkZoRoL", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:57:22.450Z", - "service": { - "type": "aws" - }, - "aws": { - "ebs": { - "metrics": { - "VolumeReadOps": { - "avg": 0 - }, - "VolumeQueueLength": { - "avg": 0.0000666666666666667 - }, - "VolumeWriteOps": { - "avg": 29 - }, - "VolumeTotalWriteTime": { - "sum": 0.02 - }, - "BurstBalance": { - "avg": 100 - }, - "VolumeWriteBytes": { - "avg": 14406.620689655172 - }, - "VolumeIdleTime": { - "sum": 299.98 - } + "@timestamp": "2020-05-28T17:57:22.450Z", + "service": { + "type": "aws" + }, + "aws": { + "ebs": { + "metrics": { + "VolumeReadOps": { + "avg": 0 + }, + "VolumeQueueLength": { + "avg": 0.0000666666666666667 + }, + "VolumeWriteOps": { + "avg": 29 + }, + "VolumeTotalWriteTime": { + "sum": 0.02 + }, + "BurstBalance": { + "avg": 100 + }, + "VolumeWriteBytes": { + "avg": 14406.620689655172 + }, + "VolumeIdleTime": { + "sum": 299.98 } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-03370a204cc8b0a2f" - } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" } }, - "event": { - "dataset": "aws.ebs", - "module": "aws", - "duration": 10488314037 - }, - "metricset": { - "period": 300000, - "name": "ebs" + "cloudwatch": { + "namespace": "AWS/EBS" }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ebs" + "dimensions": { + "VolumeId": "vol-03370a204cc8b0a2f" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:57:22.450Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ebs@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.ebs", + "module": "aws", + "duration": 10488314037 }, - "sort": [ - 1590688642450 - ] + "metricset": { + "period": 300000, + "name": "ebs" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/ec2_metrics/sample_event.json b/packages/aws/data_stream/ec2_metrics/sample_event.json index 83511916967..ffdd822660f 100644 --- a/packages/aws/data_stream/ec2_metrics/sample_event.json +++ b/packages/aws/data_stream/ec2_metrics/sample_event.json @@ -1,134 +1,110 @@ { - "_index": "metrics-aws.ec2_metrics-default-000001", - "_id": "b89uXHIBpGMSUzkZHxPP", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:56:37.255Z", - "aws": { - "ec2": { - "network": { - "in": { - "packets": 448.4, - "bytes_per_sec": 103.10266666666666, - "packets_per_sec": 1.4946666666666666, - "bytes": 30930.8 - }, - "out": { - "packets": 233.6, - "bytes_per_sec": 51.754666666666665, - "packets_per_sec": 0.7786666666666666, - "bytes": 15526.4 - } + "@timestamp": "2020-05-28T17:56:37.255Z", + "aws": { + "ec2": { + "network": { + "in": { + "packets": 448.4, + "bytes_per_sec": 103.10266666666666, + "packets_per_sec": 1.4946666666666666, + "bytes": 30930.8 }, - "status": { - "check_failed": 0, - "check_failed_instance": 0, - "check_failed_system": 0 + "out": { + "packets": 233.6, + "bytes_per_sec": 51.754666666666665, + "packets_per_sec": 0.7786666666666666, + "bytes": 15526.4 + } + }, + "status": { + "check_failed": 0, + "check_failed_instance": 0, + "check_failed_system": 0 + }, + "cpu": { + "credit_usage": 0.004566, + "credit_balance": 144, + "surplus_credit_balance": 0, + "surplus_credits_charged": 0, + "total": { + "pct": 0.0999999999997574 + } + }, + "diskio": { + "read": { + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0, + "count": 0 + }, + "write": { + "count": 0, + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0 + } + }, + "instance": { + "core": { + "count": 1 + }, + "threads_per_core": 1, + "public": { + "ip": "3.122.204.80", + "dns_name": "" + }, + "private": { + "ip": "10.0.0.122", + "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" }, - "cpu": { - "credit_usage": 0.004566, - "credit_balance": 144, - "surplus_credit_balance": 0, - "surplus_credits_charged": 0, - "total": { - "pct": 0.0999999999997574 - } + "image": { + "id": "ami-0b418580298265d5c" }, - "diskio": { - "read": { - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0, - "count": 0 - }, - "write": { - "count": 0, - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0 - } + "state": { + "name": "running", + "code": 16 }, - "instance": { - "core": { - "count": 1 - }, - "threads_per_core": 1, - "public": { - "ip": "3.122.204.80", - "dns_name": "" - }, - "private": { - "ip": "10.0.0.122", - "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" - }, - "image": { - "id": "ami-0b418580298265d5c" - }, - "state": { - "name": "running", - "code": 16 - }, - "monitoring": { - "state": "disabled" - } + "monitoring": { + "state": "disabled" } } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "aws", - "duration": 23217499283, - "dataset": "aws.ec2" - }, - "metricset": { - "period": 300000, - "name": "ec2" - }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ec2_metrics" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "instance": { - "id": "i-04c1a32c2aace6b40" - }, - "machine": { - "type": "t2.micro" - }, - "availability_zone": "eu-central-1a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:56:37.255Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "aws", + "duration": 23217499283, + "dataset": "aws.ec2" + }, + "metricset": { + "period": 300000, + "name": "ec2" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ec2@/kibana-highlighted-field@" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590688597255 - ] + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "instance": { + "id": "i-04c1a32c2aace6b40" + }, + "machine": { + "type": "t2.micro" + }, + "availability_zone": "eu-central-1a" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/elb_metrics/fields/fields.yml b/packages/aws/data_stream/elb_metrics/fields/fields.yml index d1a7a324453..dd916b17f43 100644 --- a/packages/aws/data_stream/elb_metrics/fields/fields.yml +++ b/packages/aws/data_stream/elb_metrics/fields/fields.yml @@ -193,3 +193,9 @@ - name: TargetGroup type: keyword description: Filters the metric data by target group. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/elb_metrics/sample_event.json b/packages/aws/data_stream/elb_metrics/sample_event.json index 86515657dd8..d187909719f 100644 --- a/packages/aws/data_stream/elb_metrics/sample_event.json +++ b/packages/aws/data_stream/elb_metrics/sample_event.json @@ -1,87 +1,63 @@ { - "_index": "metrics-aws.elb_metrics-default-000001", - "_id": "i89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.211Z", - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "aws": { - "elb": { - "metrics": { - "EstimatedALBNewConnectionCount": { - "avg": 32 - }, - "EstimatedALBConsumedLCUs": { - "avg": 0.00035000000000000005 - }, - "EstimatedProcessedBytes": { - "avg": 967 - }, - "EstimatedALBActiveConnectionCount": { - "avg": 5 - }, - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - } + "@timestamp": "2020-05-28T17:58:30.211Z", + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "aws": { + "elb": { + "metrics": { + "EstimatedALBNewConnectionCount": { + "avg": 32 + }, + "EstimatedALBConsumedLCUs": { + "avg": 0.00035000000000000005 + }, + "EstimatedProcessedBytes": { + "avg": 967 + }, + "EstimatedALBActiveConnectionCount": { + "avg": 5 + }, + "HealthyHostCount": { + "max": 2 + }, + "UnHealthyHostCount": { + "max": 0 } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - }, - "dimensions": { - "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "metricset": { - "name": "elb", - "period": 60000 - }, - "event": { - "dataset": "aws.elb", - "module": "aws", - "duration": 15044430616 - }, - "service": { - "type": "aws" + "cloudwatch": { + "namespace": "AWS/ELB" }, - "stream": { - "type": "metrics", - "dataset": "aws.elb_metrics", - "namespace": "default" + "dimensions": { + "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.211Z" - ] + "metricset": { + "name": "elb", + "period": 60000 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.elb@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.elb", + "module": "aws", + "duration": 15044430616 }, - "sort": [ - 1590688710211 - ] + "service": { + "type": "aws" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/lambda/fields/fields.yml b/packages/aws/data_stream/lambda/fields/fields.yml index 82a59893af7..5209e0d30ec 100644 --- a/packages/aws/data_stream/lambda/fields/fields.yml +++ b/packages/aws/data_stream/lambda/fields/fields.yml @@ -58,3 +58,9 @@ - name: ProvisionedConcurrencySpilloverInvocations.sum type: long description: The number of times your function code is executed on standard concurrency when all provisioned concurrency is in use. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/lambda/sample_event.json b/packages/aws/data_stream/lambda/sample_event.json index 0a88bde60cb..b1542233bd3 100644 --- a/packages/aws/data_stream/lambda/sample_event.json +++ b/packages/aws/data_stream/lambda/sample_event.json @@ -1,77 +1,58 @@ { - "_index": "metrics-aws.lambda-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.lambda", - "namespace": "default" - }, - "service": { - "type": "aws" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "ecs": { - "version": "1.5.0" + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "dimensions": { + "FunctionName": "ec2-owner-tagger-serverless", + "Resource": "ec2-owner-tagger-serverless" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/Lambda" - }, - "dimensions": { - "FunctionName": "ec2-owner-tagger-serverless", - "Resource": "ec2-owner-tagger-serverless" - }, - "lambda": { - "metrics": { - "Duration": { - "avg": 8218.073333333334 - }, - "Errors": { - "avg": 1 - }, - "Invocations": { - "avg": 1 - }, - "Throttles": { - "avg": 0 - } + "lambda": { + "metrics": { + "Duration": { + "avg": 8218.073333333334 + }, + "Errors": { + "avg": 1 + }, + "Invocations": { + "avg": 1 + }, + "Throttles": { + "avg": 0 } } - }, - "metricset": { - "name": "dynamodb", - "period": 300000 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } \ No newline at end of file diff --git a/packages/aws/data_stream/natgateway/fields/fields.yml b/packages/aws/data_stream/natgateway/fields/fields.yml index 78ffdb3b5a6..c3e71724550 100644 --- a/packages/aws/data_stream/natgateway/fields/fields.yml +++ b/packages/aws/data_stream/natgateway/fields/fields.yml @@ -55,3 +55,9 @@ - name: ActiveConnectionCount.max type: long description: The total number of concurrent active TCP connections through the NAT gateway. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/natgateway/sample_event.json b/packages/aws/data_stream/natgateway/sample_event.json index d6b2303b937..11f136cd632 100644 --- a/packages/aws/data_stream/natgateway/sample_event.json +++ b/packages/aws/data_stream/natgateway/sample_event.json @@ -1,108 +1,84 @@ { - "_index": "metrics-aws.natgateway-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.natgateway", - "namespace": "default", - "type": "metrics" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/NATGateway" }, - "ecs": { - "version": "1.5.0" + "dimensions": { + "NatGatewayId": "nat-0a5cb7b9807908cc0" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/NATGateway" - }, - "dimensions": { - "NatGatewayId": "nat-0a5cb7b9807908cc0" - }, - "natgateway": { - "metrics": { - "ActiveConnectionCount": { - "max": 0 - }, - "BytesInFromDestination": { - "sum": 0 - }, - "BytesInFromSource": { - "sum": 0 - }, - "BytesOutToDestination": { - "sum": 0 - }, - "BytesOutToSource": { - "sum": 0 - }, - "ConnectionAttemptCount": { - "sum": 0 - }, - "ConnectionEstablishedCount": { - "sum": 0 - }, - "ErrorPortAllocation": { - "sum": 0 - }, - "PacketsDropCount": { - "sum": 0 - }, - "PacketsInFromDestination": { - "sum": 0 - }, - "PacketsInFromSource": { - "sum": 0 - }, - "PacketsOutToDestination": { - "sum": 0 - }, - "PacketsOutToSource": { - "sum": 0 - } + "natgateway": { + "metrics": { + "ActiveConnectionCount": { + "max": 0 + }, + "BytesInFromDestination": { + "sum": 0 + }, + "BytesInFromSource": { + "sum": 0 + }, + "BytesOutToDestination": { + "sum": 0 + }, + "BytesOutToSource": { + "sum": 0 + }, + "ConnectionAttemptCount": { + "sum": 0 + }, + "ConnectionEstablishedCount": { + "sum": 0 + }, + "ErrorPortAllocation": { + "sum": 0 + }, + "PacketsDropCount": { + "sum": 0 + }, + "PacketsInFromDestination": { + "sum": 0 + }, + "PacketsInFromSource": { + "sum": 0 + }, + "PacketsOutToDestination": { + "sum": 0 + }, + "PacketsOutToSource": { + "sum": 0 } } - }, - "event": { - "dataset": "aws.natgateway", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "natgateway" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.natgateway", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.natgateway@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "natgateway" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/rds/sample_event.json b/packages/aws/data_stream/rds/sample_event.json index cb71045ad1f..27bfc3c0bf5 100644 --- a/packages/aws/data_stream/rds/sample_event.json +++ b/packages/aws/data_stream/rds/sample_event.json @@ -1,113 +1,89 @@ { - "_index": "metrics-aws.rds-default-000001", - "_id": "k89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:34.537Z", - "ecs": { - "version": "1.5.0" - }, - "service": { - "type": "aws" - }, - "aws": { - "rds": { - "latency": { - "dml": 0, - "insert": 0, - "update": 0, - "commit": 0, - "ddl": 0, - "delete": 0, - "select": 0.21927814569536422 - }, - "queries": 6.197934021992669, - "aurora_bin_log_replica_lag": 0, - "transactions": { - "blocked": 0, - "active": 0 - }, - "deadlocks": 0, - "login_failures": 0, - "throughput": { - "network": 1.399813358218904, - "insert": 0, - "ddl": 0, - "select": 2.5165408396246853, - "delete": 0, - "commit": 0, - "network_transmit": 0.699906679109452, - "update": 0, - "dml": 0, - "network_receive": 0.699906679109452 - }, - "cpu": { - "total": { - "pct": 0.03 - } - }, - "db_instance": { - "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", - "class": "db.r5.large", - "identifier": "database-1-instance-1-eu-west-1a", - "status": "available" - }, - "cache_hit_ratio.result_set": 0, - "aurora_replica.lag.ms": 19.576, - "free_local_storage.bytes": 32431271936, - "cache_hit_ratio.buffer": 100, - "disk_usage": { - "bin_log.bytes": 0 - }, - "db_instance.identifier": "database-1-instance-1-eu-west-1a", - "freeable_memory.bytes": 4436537344, - "engine_uptime.sec": 10463030, - "database_connections": 0 - } - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" + "@timestamp": "2020-05-28T17:58:34.537Z", + "ecs": { + "version": "1.5.0" + }, + "service": { + "type": "aws" + }, + "aws": { + "rds": { + "latency": { + "dml": 0, + "insert": 0, + "update": 0, + "commit": 0, + "ddl": 0, + "delete": 0, + "select": 0.21927814569536422 }, - "availability_zone": "eu-west-1a" - }, - "event": { - "dataset": "aws.rds", - "module": "aws", - "duration": 10777919184 - }, - "metricset": { - "name": "rds", - "period": 60000 - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.rds" - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + "queries": 6.197934021992669, + "aurora_bin_log_replica_lag": 0, + "transactions": { + "blocked": 0, + "active": 0 + }, + "deadlocks": 0, + "login_failures": 0, + "throughput": { + "network": 1.399813358218904, + "insert": 0, + "ddl": 0, + "select": 2.5165408396246853, + "delete": 0, + "commit": 0, + "network_transmit": 0.699906679109452, + "update": 0, + "dml": 0, + "network_receive": 0.699906679109452 + }, + "cpu": { + "total": { + "pct": 0.03 + } + }, + "db_instance": { + "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", + "class": "db.r5.large", + "identifier": "database-1-instance-1-eu-west-1a", + "status": "available" + }, + "cache_hit_ratio.result_set": 0, + "aurora_replica.lag.ms": 19.576, + "free_local_storage.bytes": 32431271936, + "cache_hit_ratio.buffer": 100, + "disk_usage": { + "bin_log.bytes": 0 + }, + "db_instance.identifier": "database-1-instance-1-eu-west-1a", + "freeable_memory.bytes": 4436537344, + "engine_uptime.sec": 10463030, + "database_connections": 0 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:34.537Z" - ] + "cloud": { + "provider": "aws", + "region": "eu-west-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "availability_zone": "eu-west-1a" + }, + "event": { + "dataset": "aws.rds", + "module": "aws", + "duration": 10777919184 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.rds@/kibana-highlighted-field@" - ] + "metricset": { + "name": "rds", + "period": 60000 }, - "sort": [ - 1590688714537 - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/s3_daily_storage/sample_event.json b/packages/aws/data_stream/s3_daily_storage/sample_event.json index b288a47a793..f3e230ff06b 100644 --- a/packages/aws/data_stream/s3_daily_storage/sample_event.json +++ b/packages/aws/data_stream/s3_daily_storage/sample_event.json @@ -1,72 +1,48 @@ { - "_index": "metrics-aws.s3_daily_storage-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_daily_storage", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_daily_storage": { - "bucket": { - "size": { - "bytes": 207372 - } - }, - "number_of_objects": 128 + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_daily_storage", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_daily_storage" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_daily_storage": { + "bucket": { + "size": { + "bytes": 207372 + } }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "number_of_objects": 128 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_daily_storage", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_daily_storage" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_daily_storage@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/s3_request/sample_event.json b/packages/aws/data_stream/s3_request/sample_event.json index 9a8d1d70139..3d1822e57ce 100644 --- a/packages/aws/data_stream/s3_request/sample_event.json +++ b/packages/aws/data_stream/s3_request/sample_event.json @@ -1,85 +1,61 @@ { - "_index": "metrics-aws.s3_request-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_request", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_request": { - "downloaded": { - "bytes": 534 - }, - "errors": { - "4xx": 0, - "5xx": 0 - }, - "latency": { - "first_byte.ms": 214, - "total_request.ms": 533 - }, - "requests": { - "list": 2, - "put": 10, - "total": 12 - }, - "uploaded": { - "bytes": 13572 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_request", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_request" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_request": { + "downloaded": { + "bytes": 534 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte.ms": 214, + "total_request.ms": 533 + }, + "requests": { + "list": 2, + "put": 10, + "total": 12 + }, + "uploaded": { + "bytes": 13572 + } } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_request", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_request" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_request@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/sns/fields/fields.yml b/packages/aws/data_stream/sns/fields/fields.yml index b3dd6000536..c07522553d3 100644 --- a/packages/aws/data_stream/sns/fields/fields.yml +++ b/packages/aws/data_stream/sns/fields/fields.yml @@ -61,3 +61,9 @@ - name: SMSMonthToDateSpentUSD.sum type: long description: The charges you have accrued since the start of the current calendar month for sending SMS messages. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/sns/sample_event.json b/packages/aws/data_stream/sns/sample_event.json index dee8ff18a5c..af48ae9aa41 100644 --- a/packages/aws/data_stream/sns/sample_event.json +++ b/packages/aws/data_stream/sns/sample_event.json @@ -1,81 +1,57 @@ { - "_index": "metrics-aws.sns-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.sns", - "namespace": "default", - "type": "metrics" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/SNS" }, - "ecs": { - "version": "1.5.0" + "dimensions": { + "TopicName": "test-sns-ks" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/SNS" - }, - "dimensions": { - "TopicName": "test-sns-ks" - }, - "sns": { - "metrics": { - "NumberOfMessagesPublished": { - "sum": 1 - }, - "NumberOfNotificationsFailed": { - "sum": 1 - }, - "PublishSize": { - "avg": 5 - } + "sns": { + "metrics": { + "NumberOfMessagesPublished": { + "sum": 1 + }, + "NumberOfNotificationsFailed": { + "sum": 1 + }, + "PublishSize": { + "avg": 5 } - }, - "tags": { - "created-by": "ks" } }, - "event": { - "dataset": "aws.sns", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sns" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "tags": { + "created-by": "ks" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.sns", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.sns@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "sns" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/sqs/sample_event.json b/packages/aws/data_stream/sqs/sample_event.json index f59384aed64..714ab645a48 100644 --- a/packages/aws/data_stream/sqs/sample_event.json +++ b/packages/aws/data_stream/sqs/sample_event.json @@ -1,77 +1,53 @@ { - "_index": "metrics-aws.sqs-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.sqs", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "sqs": { - "empty_receives": 0, - "messages": { - "delayed": 0, - "deleted": 0, - "not_visible": 0, - "received": 0, - "sent": 0, - "visible": 2 - }, - "oldest_message_age": { - "sec": 78494 - }, - "queue": { - "name": "test-s3-notification" - }, - "sent_message_size": {} - } - }, - "event": { - "dataset": "aws.sqs", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sqs" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "sqs": { + "empty_receives": 0, + "messages": { + "delayed": 0, + "deleted": 0, + "not_visible": 0, + "received": 0, + "sent": 0, + "visible": 2 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "oldest_message_age": { + "sec": 78494 + }, + "queue": { + "name": "test-s3-notification" + }, + "sent_message_size": {} } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.sqs", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "sqs" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.sqs@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/transitgateway/fields/fields.yml b/packages/aws/data_stream/transitgateway/fields/fields.yml index f1c33619689..e89af409408 100644 --- a/packages/aws/data_stream/transitgateway/fields/fields.yml +++ b/packages/aws/data_stream/transitgateway/fields/fields.yml @@ -34,3 +34,9 @@ - name: PacketDropCountNoRoute.sum type: long description: The number of packets dropped because they did not match a route. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/transitgateway/sample_event.json b/packages/aws/data_stream/transitgateway/sample_event.json index d7c4ebda322..3f9d5b46eab 100644 --- a/packages/aws/data_stream/transitgateway/sample_event.json +++ b/packages/aws/data_stream/transitgateway/sample_event.json @@ -1,87 +1,63 @@ { - "_index": "metrics-aws.transitgateway-default-000001", - "_id": "WNToXHIBpGMSUzkZaeVh", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T20:10:20.953Z", - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "transitgateway": { - "metrics": { - "PacketsIn": { - "sum": 0 - }, - "BytesIn": { - "sum": 0 - }, - "BytesOut": { - "sum": 0 - }, - "PacketsOut": { - "sum": 0 - }, - "PacketDropCountBlackhole": { - "sum": 0 - }, - "PacketDropCountNoRoute": { - "sum": 0 - } + "@timestamp": "2020-05-28T20:10:20.953Z", + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "transitgateway": { + "metrics": { + "PacketsIn": { + "sum": 0 + }, + "BytesIn": { + "sum": 0 + }, + "BytesOut": { + "sum": 0 + }, + "PacketsOut": { + "sum": 0 + }, + "PacketDropCountBlackhole": { + "sum": 0 + }, + "PacketDropCountNoRoute": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/TransitGateway" - }, - "dimensions": { - "TransitGateway": "tgw-0630672a32f12808a" } }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + "cloudwatch": { + "namespace": "AWS/TransitGateway" }, - "event": { - "dataset": "aws.transitgateway", - "module": "aws", - "duration": 12762825681 - }, - "metricset": { - "period": 60000, - "name": "transitgateway" - }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.transitgateway" + "dimensions": { + "TransitGateway": "tgw-0630672a32f12808a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T20:10:20.953Z" - ] + "ecs": { + "version": "1.5.0" + }, + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "event": { + "dataset": "aws.transitgateway", + "module": "aws", + "duration": 12762825681 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.transitgateway@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "transitgateway" }, - "sort": [ - 1590696620953 - ] + "service": { + "type": "aws" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/usage/fields/fields.yml b/packages/aws/data_stream/usage/fields/fields.yml index bb75496faaf..7cd5c5e37b2 100644 --- a/packages/aws/data_stream/usage/fields/fields.yml +++ b/packages/aws/data_stream/usage/fields/fields.yml @@ -28,3 +28,9 @@ - name: ResourceCount.sum type: long description: The number of the specified resources running in your account. The resources are defined by the dimensions associated with the metric. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/usage/sample_event.json b/packages/aws/data_stream/usage/sample_event.json index da51bc44165..c67701e60f0 100644 --- a/packages/aws/data_stream/usage/sample_event.json +++ b/packages/aws/data_stream/usage/sample_event.json @@ -1,75 +1,51 @@ { - "_index": "metrics-aws.usage-default-000001", - "_id": "YM9vXHIBpGMSUzkZiSlC", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.929Z", - "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } + "@timestamp": "2020-05-28T17:58:30.929Z", + "aws": { + "usage": { + "metrics": { + "CallCount": { + "sum": 1 } - }, - "cloudwatch": { - "namespace": "AWS/Usage" - }, - "dimensions": { - "Type": "API", - "Resource": "GetMetricData", - "Service": "CloudWatch", - "Class": "None" } }, - "event": { - "duration": 1191329839, - "dataset": "aws.usage", - "module": "aws" + "cloudwatch": { + "namespace": "AWS/Usage" }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.usage", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "metricset": { - "name": "usage", - "period": 60000 - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" + "dimensions": { + "Type": "API", + "Resource": "GetMetricData", + "Service": "CloudWatch", + "Class": "None" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.929Z" - ] + "event": { + "duration": 1191329839, + "dataset": "aws.usage", + "module": "aws" + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-north-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.usage@/kibana-highlighted-field@" - ] + "metricset": { + "name": "usage", + "period": 60000 }, - "sort": [ - 1590688710929 - ] + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + } } \ No newline at end of file diff --git a/packages/aws/data_stream/vpn/fields/fields.yml b/packages/aws/data_stream/vpn/fields/fields.yml index f8cbeb71f63..5a5ff461f09 100644 --- a/packages/aws/data_stream/vpn/fields/fields.yml +++ b/packages/aws/data_stream/vpn/fields/fields.yml @@ -25,3 +25,9 @@ - name: TunnelIpAddress type: keyword description: Filters the metric data by the IP address of the tunnel for the virtual private gateway. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/packages/aws/data_stream/vpn/sample_event.json b/packages/aws/data_stream/vpn/sample_event.json index 4a8aab803b3..a5f331f9c5e 100644 --- a/packages/aws/data_stream/vpn/sample_event.json +++ b/packages/aws/data_stream/vpn/sample_event.json @@ -1,75 +1,51 @@ { - "_index": "metrics-aws.vpn-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.vpn", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "vpn": { - "metrics": { - "TunnelState": { - "avg": 0 - }, - "TunnelDataIn": { - "sum": 0 - }, - "TunnelDataOut": { - "sum": 0 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "vpn": { + "metrics": { + "TunnelState": { + "avg": 0 + }, + "TunnelDataIn": { + "sum": 0 + }, + "TunnelDataOut": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/VPN" } }, - "event": { - "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "vpn" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "cloudwatch": { + "namespace": "AWS/VPN" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.vpn", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.vpn@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "vpn" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/aws/docs/README.md b/packages/aws/docs/README.md index f100160f819..f064316fe54 100644 --- a/packages/aws/docs/README.md +++ b/packages/aws/docs/README.md @@ -624,71 +624,52 @@ An example event for `billing` looks as following: ```$json { - "_index": "metrics-aws.billing-default-000001", - "_id": "IMxJXHIBpGMSUzkZo-s0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } + "@timestamp": "2020-05-28T17:17:06.212Z", + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.billing", + "module": "aws", + "duration": 1938760247 + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "billing": { + "metrics": { + "EstimatedCharges": { + "max": 1625.41 } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" } }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.billing", - "namespace": "default" + "cloudwatch": { + "namespace": "AWS/Billing" }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + "dimensions": { + "Currency": "USD" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:06.212Z" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590686226212 - ] + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + } } ``` @@ -716,6 +697,7 @@ An example event for `billing` looks as following: | aws.billing.group_definition.key | The string that represents a key for a specified group. | keyword | | aws.billing.group_definition.type | The string that represents the type of group. | keyword | | aws.billing.start_date | Start date for retrieving AWS costs. | keyword | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.* | Tag key value pairs from aws resources. | object | @@ -762,76 +744,57 @@ An example event for `cloudwatch` looks as following: ```$json { - "_index": "metrics-aws.cloudwatch_metrics-default-000001", - "_id": "-sxJXHIBpGMSUzkZxex8", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "stream": { - "dataset": "aws.cloudwatch_metrics", - "namespace": "default", - "type": "metrics" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "dimensions": { - "InstanceId": "i-0830bfecfa7173cbe" - }, - "ec2": { - "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, - "CPUUtilization": { - "avg": 0.7661943132361363, - "max": 0.833333333333333 - } + "@timestamp": "2020-05-28T17:17:02.812Z", + "event": { + "duration": 14119105951, + "dataset": "aws.cloudwatch", + "module": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "dimensions": { + "InstanceId": "i-0830bfecfa7173cbe" + }, + "ec2": { + "metrics": { + "DiskWriteOps": { + "avg": 0, + "max": 0 + }, + "CPUUtilization": { + "avg": 0.7661943132361363, + "max": 0.833333333333333 } - }, - "cloudwatch": { - "namespace": "AWS/EC2" } }, - "metricset": { - "period": 300000, - "name": "cloudwatch" + "cloudwatch": { + "namespace": "AWS/EC2" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:02.812Z" - ] - }, - "sort": [ - 1590686222812 - ] + "metricset": { + "period": 300000, + "name": "cloudwatch" + } } ``` @@ -888,82 +851,63 @@ An example event for `dynamodb` looks as following: ```$json { - "_index": "metrics-aws.dynamodb-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.dynamodb", - "namespace": "default" - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "aws": { - "dimensions": { - "TableName": "TryDaxTable3" - }, - "dynamodb": { - "metrics": { - "ProvisionedWriteCapacityUnits": { - "avg": 1 - }, - "ProvisionedReadCapacityUnits": { - "avg": 1 - }, - "ConsumedWriteCapacityUnits": { - "avg": 0, - "sum": 0 - }, - "ConsumedReadCapacityUnits": { - "avg": 0, - "sum": 0 - } + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "dimensions": { + "TableName": "TryDaxTable3" + }, + "dynamodb": { + "metrics": { + "ProvisionedWriteCapacityUnits": { + "avg": 1 + }, + "ProvisionedReadCapacityUnits": { + "avg": 1 + }, + "ConsumedWriteCapacityUnits": { + "avg": 0, + "sum": 0 + }, + "ConsumedReadCapacityUnits": { + "avg": 0, + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/DynamoDB" } }, - "metricset": { - "name": "dynamodb", - "period": 300000 + "cloudwatch": { + "namespace": "AWS/DynamoDB" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } ``` @@ -973,6 +917,7 @@ An example event for `dynamodb` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dynamodb.metrics.AccountMaxReads.max | The maximum number of read capacity units that can be used by an account. This limit does not apply to on-demand tables or global secondary indexes. | long | | aws.dynamodb.metrics.AccountMaxTableLevelReads.max | The maximum number of read capacity units that can be used by a table or global secondary index of an account. For on-demand tables this limit caps the maximum read request units a table or a global secondary index can use. | long | @@ -1046,94 +991,70 @@ An example event for `ebs` looks as following: ```$json { - "_index": "metrics-aws.ebs-default-000001", - "_id": "_89uXHIBpGMSUzkZoRoL", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:57:22.450Z", - "service": { - "type": "aws" - }, - "aws": { - "ebs": { - "metrics": { - "VolumeReadOps": { - "avg": 0 - }, - "VolumeQueueLength": { - "avg": 0.0000666666666666667 - }, - "VolumeWriteOps": { - "avg": 29 - }, - "VolumeTotalWriteTime": { - "sum": 0.02 - }, - "BurstBalance": { - "avg": 100 - }, - "VolumeWriteBytes": { - "avg": 14406.620689655172 - }, - "VolumeIdleTime": { - "sum": 299.98 - } + "@timestamp": "2020-05-28T17:57:22.450Z", + "service": { + "type": "aws" + }, + "aws": { + "ebs": { + "metrics": { + "VolumeReadOps": { + "avg": 0 + }, + "VolumeQueueLength": { + "avg": 0.0000666666666666667 + }, + "VolumeWriteOps": { + "avg": 29 + }, + "VolumeTotalWriteTime": { + "sum": 0.02 + }, + "BurstBalance": { + "avg": 100 + }, + "VolumeWriteBytes": { + "avg": 14406.620689655172 + }, + "VolumeIdleTime": { + "sum": 299.98 } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-03370a204cc8b0a2f" } }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + "cloudwatch": { + "namespace": "AWS/EBS" }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.ebs", - "module": "aws", - "duration": 10488314037 - }, - "metricset": { - "period": 300000, - "name": "ebs" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ebs" + "dimensions": { + "VolumeId": "vol-03370a204cc8b0a2f" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:57:22.450Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ebs@/kibana-highlighted-field@" - ] + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.ebs", + "module": "aws", + "duration": 10488314037 }, - "sort": [ - 1590688642450 - ] + "metricset": { + "period": 300000, + "name": "ebs" + } } ``` @@ -1143,6 +1064,7 @@ An example event for `ebs` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.VolumeId | Amazon EBS volume ID | keyword | | aws.ebs.metrics.BurstBalance.avg | Used with General Purpose SSD (gp2), Throughput Optimized HDD (st1), and Cold HDD (sc1) volumes only. Provides information about the percentage of I/O credits (for gp2) or throughput credits (for st1 and sc1) remaining in the burst bucket. | double | @@ -1201,138 +1123,114 @@ An example event for `ec2` looks as following: ```$json { - "_index": "metrics-aws.ec2_metrics-default-000001", - "_id": "b89uXHIBpGMSUzkZHxPP", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:56:37.255Z", - "aws": { - "ec2": { - "network": { - "in": { - "packets": 448.4, - "bytes_per_sec": 103.10266666666666, - "packets_per_sec": 1.4946666666666666, - "bytes": 30930.8 - }, - "out": { - "packets": 233.6, - "bytes_per_sec": 51.754666666666665, - "packets_per_sec": 0.7786666666666666, - "bytes": 15526.4 - } + "@timestamp": "2020-05-28T17:56:37.255Z", + "aws": { + "ec2": { + "network": { + "in": { + "packets": 448.4, + "bytes_per_sec": 103.10266666666666, + "packets_per_sec": 1.4946666666666666, + "bytes": 30930.8 }, - "status": { - "check_failed": 0, - "check_failed_instance": 0, - "check_failed_system": 0 + "out": { + "packets": 233.6, + "bytes_per_sec": 51.754666666666665, + "packets_per_sec": 0.7786666666666666, + "bytes": 15526.4 + } + }, + "status": { + "check_failed": 0, + "check_failed_instance": 0, + "check_failed_system": 0 + }, + "cpu": { + "credit_usage": 0.004566, + "credit_balance": 144, + "surplus_credit_balance": 0, + "surplus_credits_charged": 0, + "total": { + "pct": 0.0999999999997574 + } + }, + "diskio": { + "read": { + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0, + "count": 0 }, - "cpu": { - "credit_usage": 0.004566, - "credit_balance": 144, - "surplus_credit_balance": 0, - "surplus_credits_charged": 0, - "total": { - "pct": 0.0999999999997574 - } + "write": { + "count": 0, + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0 + } + }, + "instance": { + "core": { + "count": 1 + }, + "threads_per_core": 1, + "public": { + "ip": "3.122.204.80", + "dns_name": "" + }, + "private": { + "ip": "10.0.0.122", + "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" + }, + "image": { + "id": "ami-0b418580298265d5c" }, - "diskio": { - "read": { - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0, - "count": 0 - }, - "write": { - "count": 0, - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0 - } + "state": { + "name": "running", + "code": 16 }, - "instance": { - "core": { - "count": 1 - }, - "threads_per_core": 1, - "public": { - "ip": "3.122.204.80", - "dns_name": "" - }, - "private": { - "ip": "10.0.0.122", - "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" - }, - "image": { - "id": "ami-0b418580298265d5c" - }, - "state": { - "name": "running", - "code": 16 - }, - "monitoring": { - "state": "disabled" - } + "monitoring": { + "state": "disabled" } } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "aws", - "duration": 23217499283, - "dataset": "aws.ec2" - }, - "metricset": { - "period": 300000, - "name": "ec2" - }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ec2_metrics" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "instance": { - "id": "i-04c1a32c2aace6b40" - }, - "machine": { - "type": "t2.micro" - }, - "availability_zone": "eu-central-1a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:56:37.255Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "aws", + "duration": 23217499283, + "dataset": "aws.ec2" + }, + "metricset": { + "period": 300000, + "name": "ec2" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ec2@/kibana-highlighted-field@" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590688597255 - ] + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "instance": { + "id": "i-04c1a32c2aace6b40" + }, + "machine": { + "type": "t2.micro" + }, + "availability_zone": "eu-central-1a" + } } ``` @@ -1433,91 +1331,67 @@ An example event for `elb` looks as following: ```$json { - "_index": "metrics-aws.elb_metrics-default-000001", - "_id": "i89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.211Z", - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "aws": { - "elb": { - "metrics": { - "EstimatedALBNewConnectionCount": { - "avg": 32 - }, - "EstimatedALBConsumedLCUs": { - "avg": 0.00035000000000000005 - }, - "EstimatedProcessedBytes": { - "avg": 967 - }, - "EstimatedALBActiveConnectionCount": { - "avg": 5 - }, - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - } + "@timestamp": "2020-05-28T17:58:30.211Z", + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "aws": { + "elb": { + "metrics": { + "EstimatedALBNewConnectionCount": { + "avg": 32 + }, + "EstimatedALBConsumedLCUs": { + "avg": 0.00035000000000000005 + }, + "EstimatedProcessedBytes": { + "avg": 967 + }, + "EstimatedALBActiveConnectionCount": { + "avg": 5 + }, + "HealthyHostCount": { + "max": 2 + }, + "UnHealthyHostCount": { + "max": 0 } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - }, - "dimensions": { - "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "metricset": { - "name": "elb", - "period": 60000 + "cloudwatch": { + "namespace": "AWS/ELB" }, - "event": { - "dataset": "aws.elb", - "module": "aws", - "duration": 15044430616 - }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.elb_metrics", - "namespace": "default" + "dimensions": { + "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.211Z" - ] + "metricset": { + "name": "elb", + "period": 60000 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.elb@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.elb", + "module": "aws", + "duration": 15044430616 }, - "sort": [ - 1590688710211 - ] + "service": { + "type": "aws" + } } ``` @@ -1547,6 +1421,7 @@ An example event for `elb` looks as following: | aws.applicationelb.metrics.RejectedConnectionCount.sum | The number of connections that were rejected because the load balancer had reached its maximum number of connections. | long | | aws.applicationelb.metrics.RequestCount.sum | The number of requests processed over IPv4 and IPv6. | long | | aws.applicationelb.metrics.RuleEvaluations.sum | The number of rules processed by the load balancer given a request rate averaged over an hour. | long | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.AvailabilityZone | Filters the metric data by the specified Availability Zone. | keyword | | aws.dimensions.LoadBalancer | Filters the metric data by load balancer. | keyword | @@ -1630,81 +1505,62 @@ An example event for `lambda` looks as following: ```$json { - "_index": "metrics-aws.lambda-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.lambda", - "namespace": "default" - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/Lambda" - }, - "dimensions": { - "FunctionName": "ec2-owner-tagger-serverless", - "Resource": "ec2-owner-tagger-serverless" - }, - "lambda": { - "metrics": { - "Duration": { - "avg": 8218.073333333334 - }, - "Errors": { - "avg": 1 - }, - "Invocations": { - "avg": 1 - }, - "Throttles": { - "avg": 0 - } + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" + }, + "dimensions": { + "FunctionName": "ec2-owner-tagger-serverless", + "Resource": "ec2-owner-tagger-serverless" + }, + "lambda": { + "metrics": { + "Duration": { + "avg": 8218.073333333334 + }, + "Errors": { + "avg": 1 + }, + "Invocations": { + "avg": 1 + }, + "Throttles": { + "avg": 0 } } - }, - "metricset": { - "name": "dynamodb", - "period": 300000 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } ``` @@ -1714,6 +1570,7 @@ An example event for `lambda` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.ExecutedVersion | Use the ExecutedVersion dimension to compare error rates for two versions of a function that are both targets of a weighted alias. | keyword | | aws.dimensions.FunctionName | Lambda function name. | keyword | @@ -1776,112 +1633,88 @@ An example event for `natgateway` looks as following: ```$json { - "_index": "metrics-aws.natgateway-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.natgateway", - "namespace": "default", - "type": "metrics" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/NATGateway" }, - "ecs": { - "version": "1.5.0" + "dimensions": { + "NatGatewayId": "nat-0a5cb7b9807908cc0" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/NATGateway" - }, - "dimensions": { - "NatGatewayId": "nat-0a5cb7b9807908cc0" - }, - "natgateway": { - "metrics": { - "ActiveConnectionCount": { - "max": 0 - }, - "BytesInFromDestination": { - "sum": 0 - }, - "BytesInFromSource": { - "sum": 0 - }, - "BytesOutToDestination": { - "sum": 0 - }, - "BytesOutToSource": { - "sum": 0 - }, - "ConnectionAttemptCount": { - "sum": 0 - }, - "ConnectionEstablishedCount": { - "sum": 0 - }, - "ErrorPortAllocation": { - "sum": 0 - }, - "PacketsDropCount": { - "sum": 0 - }, - "PacketsInFromDestination": { - "sum": 0 - }, - "PacketsInFromSource": { - "sum": 0 - }, - "PacketsOutToDestination": { - "sum": 0 - }, - "PacketsOutToSource": { - "sum": 0 - } + "natgateway": { + "metrics": { + "ActiveConnectionCount": { + "max": 0 + }, + "BytesInFromDestination": { + "sum": 0 + }, + "BytesInFromSource": { + "sum": 0 + }, + "BytesOutToDestination": { + "sum": 0 + }, + "BytesOutToSource": { + "sum": 0 + }, + "ConnectionAttemptCount": { + "sum": 0 + }, + "ConnectionEstablishedCount": { + "sum": 0 + }, + "ErrorPortAllocation": { + "sum": 0 + }, + "PacketsDropCount": { + "sum": 0 + }, + "PacketsInFromDestination": { + "sum": 0 + }, + "PacketsInFromSource": { + "sum": 0 + }, + "PacketsOutToDestination": { + "sum": 0 + }, + "PacketsOutToSource": { + "sum": 0 } } - }, - "event": { - "dataset": "aws.natgateway", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "natgateway" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.natgateway", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.natgateway@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "natgateway" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -1891,6 +1724,7 @@ An example event for `natgateway` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.NatGatewayId | Filter the metric data by the NAT gateway ID. | keyword | | aws.natgateway.metrics.ActiveConnectionCount.max | The total number of concurrent active TCP connections through the NAT gateway. | long | @@ -1952,117 +1786,93 @@ An example event for `rds` looks as following: ```$json { - "_index": "metrics-aws.rds-default-000001", - "_id": "k89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:34.537Z", - "ecs": { - "version": "1.5.0" - }, - "service": { - "type": "aws" - }, - "aws": { - "rds": { - "latency": { - "dml": 0, - "insert": 0, - "update": 0, - "commit": 0, - "ddl": 0, - "delete": 0, - "select": 0.21927814569536422 - }, - "queries": 6.197934021992669, - "aurora_bin_log_replica_lag": 0, - "transactions": { - "blocked": 0, - "active": 0 - }, - "deadlocks": 0, - "login_failures": 0, - "throughput": { - "network": 1.399813358218904, - "insert": 0, - "ddl": 0, - "select": 2.5165408396246853, - "delete": 0, - "commit": 0, - "network_transmit": 0.699906679109452, - "update": 0, - "dml": 0, - "network_receive": 0.699906679109452 - }, - "cpu": { - "total": { - "pct": 0.03 - } - }, - "db_instance": { - "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", - "class": "db.r5.large", - "identifier": "database-1-instance-1-eu-west-1a", - "status": "available" - }, - "cache_hit_ratio.result_set": 0, - "aurora_replica.lag.ms": 19.576, - "free_local_storage.bytes": 32431271936, - "cache_hit_ratio.buffer": 100, - "disk_usage": { - "bin_log.bytes": 0 - }, - "db_instance.identifier": "database-1-instance-1-eu-west-1a", - "freeable_memory.bytes": 4436537344, - "engine_uptime.sec": 10463030, - "database_connections": 0 - } - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" + "@timestamp": "2020-05-28T17:58:34.537Z", + "ecs": { + "version": "1.5.0" + }, + "service": { + "type": "aws" + }, + "aws": { + "rds": { + "latency": { + "dml": 0, + "insert": 0, + "update": 0, + "commit": 0, + "ddl": 0, + "delete": 0, + "select": 0.21927814569536422 }, - "availability_zone": "eu-west-1a" - }, - "event": { - "dataset": "aws.rds", - "module": "aws", - "duration": 10777919184 - }, - "metricset": { - "name": "rds", - "period": 60000 - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.rds" - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + "queries": 6.197934021992669, + "aurora_bin_log_replica_lag": 0, + "transactions": { + "blocked": 0, + "active": 0 + }, + "deadlocks": 0, + "login_failures": 0, + "throughput": { + "network": 1.399813358218904, + "insert": 0, + "ddl": 0, + "select": 2.5165408396246853, + "delete": 0, + "commit": 0, + "network_transmit": 0.699906679109452, + "update": 0, + "dml": 0, + "network_receive": 0.699906679109452 + }, + "cpu": { + "total": { + "pct": 0.03 + } + }, + "db_instance": { + "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", + "class": "db.r5.large", + "identifier": "database-1-instance-1-eu-west-1a", + "status": "available" + }, + "cache_hit_ratio.result_set": 0, + "aurora_replica.lag.ms": 19.576, + "free_local_storage.bytes": 32431271936, + "cache_hit_ratio.buffer": 100, + "disk_usage": { + "bin_log.bytes": 0 + }, + "db_instance.identifier": "database-1-instance-1-eu-west-1a", + "freeable_memory.bytes": 4436537344, + "engine_uptime.sec": 10463030, + "database_connections": 0 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:34.537Z" - ] + "cloud": { + "provider": "aws", + "region": "eu-west-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "availability_zone": "eu-west-1a" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.rds@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.rds", + "module": "aws", + "duration": 10777919184 }, - "sort": [ - 1590688714537 - ] + "metricset": { + "name": "rds", + "period": 60000 + }, + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + } } ``` @@ -2199,76 +2009,52 @@ An example event for `s3_daily_storage` looks as following: ```$json { - "_index": "metrics-aws.s3_daily_storage-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_daily_storage", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_daily_storage": { - "bucket": { - "size": { - "bytes": 207372 - } - }, - "number_of_objects": 128 + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_daily_storage", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_daily_storage" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_daily_storage": { + "bucket": { + "size": { + "bytes": 207372 + } }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "number_of_objects": 128 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_daily_storage", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_daily_storage@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "s3_daily_storage" }, - "sort": [ - 1590688707154 - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2329,89 +2115,65 @@ An example event for `s3_request` looks as following: ```$json { - "_index": "metrics-aws.s3_request-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_request", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_request": { - "downloaded": { - "bytes": 534 - }, - "errors": { - "4xx": 0, - "5xx": 0 - }, - "latency": { - "first_byte.ms": 214, - "total_request.ms": 533 - }, - "requests": { - "list": 2, - "put": 10, - "total": 12 - }, - "uploaded": { - "bytes": 13572 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_request", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_request" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_request": { + "downloaded": { + "bytes": 534 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte.ms": 214, + "total_request.ms": 533 + }, + "requests": { + "list": 2, + "put": 10, + "total": 12 + }, + "uploaded": { + "bytes": 13572 + } } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_request", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_request@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "s3_request" }, - "sort": [ - 1590688707154 - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2486,85 +2248,61 @@ An example event for `sns` looks as following: ```$json { - "_index": "metrics-aws.sns-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.sns", - "namespace": "default", - "type": "metrics" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/SNS" }, - "ecs": { - "version": "1.5.0" + "dimensions": { + "TopicName": "test-sns-ks" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/SNS" - }, - "dimensions": { - "TopicName": "test-sns-ks" - }, - "sns": { - "metrics": { - "NumberOfMessagesPublished": { - "sum": 1 - }, - "NumberOfNotificationsFailed": { - "sum": 1 - }, - "PublishSize": { - "avg": 5 - } + "sns": { + "metrics": { + "NumberOfMessagesPublished": { + "sum": 1 + }, + "NumberOfNotificationsFailed": { + "sum": 1 + }, + "PublishSize": { + "avg": 5 } - }, - "tags": { - "created-by": "ks" } }, - "event": { - "dataset": "aws.sns", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sns" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "tags": { + "created-by": "ks" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.sns", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.sns@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "sns" }, - "sort": [ - 1590688707154 - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2574,6 +2312,7 @@ An example event for `sns` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.Application | Filters on application objects, which represent an app and device registered with one of the supported push notification services, such as APNs and FCM. | keyword | | aws.dimensions.Application,Platform | Filters on application and platform objects, where the platform objects are for the supported push notification services, such as APNs and FCM. | keyword | @@ -2637,81 +2376,57 @@ An example event for `sqs` looks as following: ```$json { - "_index": "metrics-aws.sqs-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.sqs", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "sqs": { - "empty_receives": 0, - "messages": { - "delayed": 0, - "deleted": 0, - "not_visible": 0, - "received": 0, - "sent": 0, - "visible": 2 - }, - "oldest_message_age": { - "sec": 78494 - }, - "queue": { - "name": "test-s3-notification" - }, - "sent_message_size": {} - } - }, - "event": { - "dataset": "aws.sqs", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sqs" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "sqs": { + "empty_receives": 0, + "messages": { + "delayed": 0, + "deleted": 0, + "not_visible": 0, + "received": 0, + "sent": 0, + "visible": 2 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "oldest_message_age": { + "sec": 78494 + }, + "queue": { + "name": "test-s3-notification" + }, + "sent_message_size": {} } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.sqs", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "sqs" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.sqs@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2778,91 +2493,67 @@ An example event for `transitgateway` looks as following: ```$json { - "_index": "metrics-aws.transitgateway-default-000001", - "_id": "WNToXHIBpGMSUzkZaeVh", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T20:10:20.953Z", - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "transitgateway": { - "metrics": { - "PacketsIn": { - "sum": 0 - }, - "BytesIn": { - "sum": 0 - }, - "BytesOut": { - "sum": 0 - }, - "PacketsOut": { - "sum": 0 - }, - "PacketDropCountBlackhole": { - "sum": 0 - }, - "PacketDropCountNoRoute": { - "sum": 0 - } + "@timestamp": "2020-05-28T20:10:20.953Z", + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "transitgateway": { + "metrics": { + "PacketsIn": { + "sum": 0 + }, + "BytesIn": { + "sum": 0 + }, + "BytesOut": { + "sum": 0 + }, + "PacketsOut": { + "sum": 0 + }, + "PacketDropCountBlackhole": { + "sum": 0 + }, + "PacketDropCountNoRoute": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/TransitGateway" - }, - "dimensions": { - "TransitGateway": "tgw-0630672a32f12808a" } }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "event": { - "dataset": "aws.transitgateway", - "module": "aws", - "duration": 12762825681 - }, - "metricset": { - "period": 60000, - "name": "transitgateway" - }, - "service": { - "type": "aws" + "cloudwatch": { + "namespace": "AWS/TransitGateway" }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.transitgateway" + "dimensions": { + "TransitGateway": "tgw-0630672a32f12808a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T20:10:20.953Z" - ] + "ecs": { + "version": "1.5.0" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.transitgateway@/kibana-highlighted-field@" - ] + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" }, - "sort": [ - 1590696620953 - ] + "event": { + "dataset": "aws.transitgateway", + "module": "aws", + "duration": 12762825681 + }, + "metricset": { + "period": 60000, + "name": "transitgateway" + }, + "service": { + "type": "aws" + } } ``` @@ -2872,6 +2563,7 @@ An example event for `transitgateway` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.TransitGateway | Filters the metric data by transit gateway. | keyword | | aws.dimensions.TransitGatewayAttachment | Filters the metric data by transit gateway attachment. | keyword | @@ -2926,79 +2618,55 @@ An example event for `usage` looks as following: ```$json { - "_index": "metrics-aws.usage-default-000001", - "_id": "YM9vXHIBpGMSUzkZiSlC", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.929Z", - "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } + "@timestamp": "2020-05-28T17:58:30.929Z", + "aws": { + "usage": { + "metrics": { + "CallCount": { + "sum": 1 } - }, - "cloudwatch": { - "namespace": "AWS/Usage" - }, - "dimensions": { - "Type": "API", - "Resource": "GetMetricData", - "Service": "CloudWatch", - "Class": "None" } }, - "event": { - "duration": 1191329839, - "dataset": "aws.usage", - "module": "aws" - }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.usage", - "namespace": "default" + "cloudwatch": { + "namespace": "AWS/Usage" }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "metricset": { - "name": "usage", - "period": 60000 - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" + "dimensions": { + "Type": "API", + "Resource": "GetMetricData", + "Service": "CloudWatch", + "Class": "None" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.929Z" - ] + "event": { + "duration": 1191329839, + "dataset": "aws.usage", + "module": "aws" + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.usage@/kibana-highlighted-field@" - ] + "cloud": { + "provider": "aws", + "region": "eu-north-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "metricset": { + "name": "usage", + "period": 60000 }, - "sort": [ - 1590688710929 - ] + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + } } ``` @@ -3008,6 +2676,7 @@ An example event for `usage` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.Class | The class of resource being tracked. | keyword | | aws.dimensions.Resource | The name of the API operation. | keyword | @@ -3060,79 +2729,55 @@ An example event for `vpn` looks as following: ```$json { - "_index": "metrics-aws.vpn-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.vpn", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "vpn": { - "metrics": { - "TunnelState": { - "avg": 0 - }, - "TunnelDataIn": { - "sum": 0 - }, - "TunnelDataOut": { - "sum": 0 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "vpn": { + "metrics": { + "TunnelState": { + "avg": 0 + }, + "TunnelDataIn": { + "sum": 0 + }, + "TunnelDataOut": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/VPN" } }, - "event": { - "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "vpn" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "cloudwatch": { + "namespace": "AWS/VPN" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.vpn", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "vpn" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.vpn@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -3142,6 +2787,7 @@ An example event for `vpn` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.TunnelIpAddress | Filters the metric data by the IP address of the tunnel for the virtual private gateway. | keyword | | aws.dimensions.VpnId | Filters the metric data by the Site-to-Site VPN connection ID. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 19379f6e4ac..7fe02e4e96b 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 0.4.0 +version: 0.4.1 license: basic description: AWS Integration type: integration From 62c14557b764cfe6b1a69f9046928aa0ded468b7 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:23:46 +0100 Subject: [PATCH 03/18] Adjust other integrations --- .../activitylogs/sample_event.json | 200 +- .../data_stream/auditlogs/sample_event.json | 132 +- .../platformlogs/sample_event.json | 148 +- .../data_stream/signinlogs/sample_event.json | 144 +- packages/azure/docs/README.md | 624 ++- .../data_stream/apiserver/sample_event.json | 146 +- .../data_stream/container/sample_event.json | 301 +- .../controllermanager/sample_event.json | 206 +- .../data_stream/event/sample_event.json | 179 +- .../data_stream/node/sample_event.json | 295 +- .../data_stream/pod/sample_event.json | 255 +- .../data_stream/proxy/sample_event.json | 436 +- .../data_stream/scheduler/sample_event.json | 182 +- .../state_container/sample_event.json | 174 +- .../state_cronjob/sample_event.json | 154 +- .../state_daemonset/sample_event.json | 144 +- .../state_deployment/sample_event.json | 144 +- .../data_stream/state_node/sample_event.json | 184 +- .../state_persistentvolume/sample_event.json | 140 +- .../sample_event.json | 142 +- .../data_stream/state_pod/sample_event.json | 156 +- .../state_replicaset/sample_event.json | 156 +- .../state_resourcequota/sample_event.json | 136 +- .../state_service/sample_event.json | 145 +- .../state_statefulset/sample_event.json | 147 +- .../state_storageclass/sample_event.json | 139 +- .../data_stream/system/sample_event.json | 221 +- .../data_stream/volume/sample_event.json | 212 +- packages/kubernetes/docs/README.md | 4410 ++++++++--------- .../data_stream/collstats/sample_event.json | 185 +- .../data_stream/dbstats/sample_event.json | 117 +- .../mongodb/data_stream/log/sample_event.json | 109 +- .../data_stream/metrics/sample_event.json | 471 +- .../data_stream/replstatus/sample_event.json | 84 +- .../data_stream/status/sample_event.json | 389 +- packages/mongodb/docs/README.md | 1246 +++-- .../data_stream/collector/sample_event.json | 81 +- .../data_stream/query/sample_event.json | 95 +- .../remote_write/sample_event.json | 82 +- packages/prometheus/docs/README.md | 258 +- .../data_stream/connection/sample_event.json | 83 +- .../data_stream/mntr/sample_event.json | 103 +- .../data_stream/server/sample_event.json | 102 +- packages/zookeeper/docs/README.md | 288 +- 44 files changed, 6336 insertions(+), 7409 deletions(-) diff --git a/packages/azure/data_stream/activitylogs/sample_event.json b/packages/azure/data_stream/activitylogs/sample_event.json index acbaab4fb83..799fe904332 100644 --- a/packages/azure/data_stream/activitylogs/sample_event.json +++ b/packages/azure/data_stream/activitylogs/sample_event.json @@ -1,111 +1,105 @@ { - "_index": ".ds-logs-azure.activitylogs-default-000001", - "_type": "_doc", - "_id": "bQlEe3UBm_qs2Y3aNZPq", - "_score": null, - "_source": { - "log": { - "level": "Information" + "log": { + "level": "Information" + }, + "azure-eventhub": { + "sequence_number": 643, + "consumer_group": "$Default", + "offset": 107374182400, + "eventhub": "insights-activity-logs", + "enqueued_time": "2020-11-02T08:59:38.905Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-02T08:51:36.997Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.activitylogs" + }, + "event": { + "duration": "0", + "ingested": "2020-10-30T20:47:48.123859400Z", + "kind": "event", + "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", + "dataset": "azure.activitylogs", + "outcome": "success" + }, + "azure": { + "subscription_id": "3f041b6d-fc31-41d8-8ff6-e5f16e6747ff", + "resource": { + "provider": "MICROSOFT.RESOURCES/DEPLOYMENTS", + "name": "NOMARKETPLACE", + "id": "/SUBSCRIPTIONS/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.RESOURCES/DEPLOYMENTS/NOMARKETPLACE", + "group": "OBS-TEST" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-activity-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-02T08:51:36.997Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.activitylogs" - }, - "event": { - "duration": "0", - "ingested": "2020-10-30T20:47:48.123859400Z", - "kind": "event", - "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", - "dataset": "azure.activitylogs", - "outcome": "success" - }, - "azure": { - "subscription_id": "3f041b6d-fc31-41d8-8ff6-e5f16e6747ff", - "resource": { - "provider": "MICROSOFT.RESOURCES/DEPLOYMENTS", - "name": "NOMARKETPLACE", - "id": "/SUBSCRIPTIONS/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.RESOURCES/DEPLOYMENTS/NOMARKETPLACE", - "group": "OBS-TEST" - }, - "correlation_id": "876190b4-5b99-4a39-b725-4f5644911cf0", - "activitylogs": { - "operation_name": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", - "result_type": "Success", - "identity": { - "authorization": { - "evidence": { - "role_definition_id": "8e3af657a8ff443ca75c2fe8c4bcb635", - "role": "Owner", - "role_assignment_scope": "/providers/Microsoft.Management/managementGroups/5341238b-665c-4eb4-b259-b250371ae430", - "role_assignment_id": "7f06f09dd6764b44930adbec3f10e92b", - "principal_type": "User", - "principal_id": "68b1adf93eb744b08eb8ce96522a08d3" - }, - "scope": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace", - "action": "Microsoft.Resources/deployments/write" - }, - "claims": { - "xms_tcdt": "1469565974", - "aio": "ATQAy/8RAAAAsL67UQMOHZv3izTDRJfvJN5UyON9ktUszzPj08K8aURsbhxhR0niz9s1Pxm9U1lI", - "iss": "https://sts.windows.net/4fa94b7d-a743-486f-abcc-6c276c44cf4b/", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "a9L2WR3XZN5ANzAqwLx_4aamU49JG6kqaE5JZkXdeNs", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Doe", - "http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", - "http://schemas_microsoft_com/identity/claims/tenantid": "4fa94b7d-a743-486f-abcc-6c276c44cf4b", - "puid": "1003200045B17AD4", - "wids": "5d6b6bb7-de71-4623-b4af-96380a352509", - "http://schemas_microsoft_com/claims/authnclassreference": "1", - "exp": "1604310019", - "ipaddr": "77.170.179.229", - "iat": "1604306119", - "http://schemas_microsoft_com/identity/claims/objectidentifier": "68b1adf9-3eb7-44b0-8eb8-ce96522a08d3", - "http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", - "ver": "1.0", - "groups": "644c6686-9ef1-4b69-9410-107664a9e1f0,9ed1993c-ce9c-4915-a04d-58c6f5f7ee12", - "uti": "rqr63RW_Kk6ztuomENMQAA", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "john@gmail.com", - "aud": "https://management.core.windows.net/", - "nbf": "1604306119", - "appidacr": "2", - "rh": "0.AAAAfUupT0Onb0irzGwnbETPS4NAS8SwO8FJtH2XTlPL3zxRAA8.", - "appid": "c44b4083-3bb0-49c1-b47d-974e53cbdf3c", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": "john@gmail.com" + "correlation_id": "876190b4-5b99-4a39-b725-4f5644911cf0", + "activitylogs": { + "operation_name": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", + "result_type": "Success", + "identity": { + "authorization": { + "evidence": { + "role_definition_id": "8e3af657a8ff443ca75c2fe8c4bcb635", + "role": "Owner", + "role_assignment_scope": "/providers/Microsoft.Management/managementGroups/5341238b-665c-4eb4-b259-b250371ae430", + "role_assignment_id": "7f06f09dd6764b44930adbec3f10e92b", + "principal_type": "User", + "principal_id": "68b1adf93eb744b08eb8ce96522a08d3" }, - "claims_initiated_by_user": { - "schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims" - } + "scope": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace", + "action": "Microsoft.Resources/deployments/write" }, - "category": "Administrative", - "event_category": "Administrative", - "result_signature": "Succeeded.", - "properties": { - "eventCategory": "Administrative", - "hierarchy": "", - "message": "Microsoft.Resources/deployments/write", - "entity": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace" + "claims": { + "xms_tcdt": "1469565974", + "aio": "ATQAy/8RAAAAsL67UQMOHZv3izTDRJfvJN5UyON9ktUszzPj08K8aURsbhxhR0niz9s1Pxm9U1lI", + "iss": "https://sts.windows.net/4fa94b7d-a743-486f-abcc-6c276c44cf4b/", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "a9L2WR3XZN5ANzAqwLx_4aamU49JG6kqaE5JZkXdeNs", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Doe", + "http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", + "http://schemas_microsoft_com/identity/claims/tenantid": "4fa94b7d-a743-486f-abcc-6c276c44cf4b", + "puid": "1003200045B17AD4", + "wids": "5d6b6bb7-de71-4623-b4af-96380a352509", + "http://schemas_microsoft_com/claims/authnclassreference": "1", + "exp": "1604310019", + "ipaddr": "77.170.179.229", + "iat": "1604306119", + "http://schemas_microsoft_com/identity/claims/objectidentifier": "68b1adf9-3eb7-44b0-8eb8-ce96522a08d3", + "http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", + "ver": "1.0", + "groups": "644c6686-9ef1-4b69-9410-107664a9e1f0,9ed1993c-ce9c-4915-a04d-58c6f5f7ee12", + "uti": "rqr63RW_Kk6ztuomENMQAA", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "john@gmail.com", + "aud": "https://management.core.windows.net/", + "nbf": "1604306119", + "appidacr": "2", + "rh": "0.AAAAfUupT0Onb0irzGwnbETPS4NAS8SwO8FJtH2XTlPL3zxRAA8.", + "appid": "c44b4083-3bb0-49c1-b47d-974e53cbdf3c", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": "john@gmail.com" + }, + "claims_initiated_by_user": { + "schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims" } + }, + "category": "Administrative", + "event_category": "Administrative", + "result_signature": "Succeeded.", + "properties": { + "eventCategory": "Administrative", + "hierarchy": "", + "message": "Microsoft.Resources/deployments/write", + "entity": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace" } } } diff --git a/packages/azure/data_stream/auditlogs/sample_event.json b/packages/azure/data_stream/auditlogs/sample_event.json index 7f05c2f8d15..c57595de91d 100644 --- a/packages/azure/data_stream/auditlogs/sample_event.json +++ b/packages/azure/data_stream/auditlogs/sample_event.json @@ -1,71 +1,65 @@ { - "_index": ".ds-logs-azure.auditlogs-default-000001", - "_type": "_doc", - "_id": "bQlEe3UBm_qs2Y3aNZPq", - "_score": null, - "_source": { - "log": { - "level": "Information" - }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-auditlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-02T08:51:36.997Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.auditlogs" - }, - "event": { - "duration": "0", - "ingested": "2020-10-30T20:47:48.123859400Z", - "kind": "event", - "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", - "dataset": "azure.auditlogs", - "outcome": "success" - }, - "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", - "azure.resource.provider": "Microsoft.aadiam", - "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.category": "AuditLogs", - "azure.auditlogs.identity": "Device Registration Service", - "azure.auditlogs.operation_name": "Update device", - "azure.auditlogs.operation_version": "1.0", - "azure.auditlogs.properties.activity_datetime": "2019-10-18T15:30:51.0273716+00:00", - "azure.auditlogs.properties.activity_display_name": "Update device", - "azure.auditlogs.properties.category": "Device", - "azure.auditlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.id": "Directory_ESQ", - "azure.auditlogs.properties.initiated_by.app.appId": null, - "azure.auditlogs.properties.initiated_by.app.displayName": "Device Registration Service", - "azure.auditlogs.properties.initiated_by.app.servicePrincipalId": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.initiated_by.app.servicePrincipalName": null, - "azure.auditlogs.properties.logged_by_service": "Core Directory", - "azure.auditlogs.properties.operation_type": "Update", - "azure.auditlogs.properties.result_reason": "", - "azure.auditlogs.properties.target_resources.0.display_name": "LAPTOP-12", - "azure.auditlogs.properties.target_resources.0.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.display_name": "Included Updated Properties", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "\"\"", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": null, - "azure.auditlogs.properties.target_resources.0.type": "Device", - "azure.auditlogs.result_signature": "None" - } + "log": { + "level": "Information" + }, + "azure-eventhub": { + "sequence_number": 643, + "consumer_group": "$Default", + "offset": 107374182400, + "eventhub": "insights-auditlogs-logs", + "enqueued_time": "2020-11-02T08:59:38.905Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-02T08:51:36.997Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.auditlogs" + }, + "event": { + "duration": "0", + "ingested": "2020-10-30T20:47:48.123859400Z", + "kind": "event", + "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", + "dataset": "azure.auditlogs", + "outcome": "success" + }, + "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", + "azure.resource.provider": "Microsoft.aadiam", + "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.category": "AuditLogs", + "azure.auditlogs.identity": "Device Registration Service", + "azure.auditlogs.operation_name": "Update device", + "azure.auditlogs.operation_version": "1.0", + "azure.auditlogs.properties.activity_datetime": "2019-10-18T15:30:51.0273716+00:00", + "azure.auditlogs.properties.activity_display_name": "Update device", + "azure.auditlogs.properties.category": "Device", + "azure.auditlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.properties.id": "Directory_ESQ", + "azure.auditlogs.properties.initiated_by.app.appId": null, + "azure.auditlogs.properties.initiated_by.app.displayName": "Device Registration Service", + "azure.auditlogs.properties.initiated_by.app.servicePrincipalId": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.properties.initiated_by.app.servicePrincipalName": null, + "azure.auditlogs.properties.logged_by_service": "Core Directory", + "azure.auditlogs.properties.operation_type": "Update", + "azure.auditlogs.properties.result_reason": "", + "azure.auditlogs.properties.target_resources.0.display_name": "LAPTOP-12", + "azure.auditlogs.properties.target_resources.0.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.properties.target_resources.0.modified_properties.0.display_name": "Included Updated Properties", + "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "\"\"", + "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": null, + "azure.auditlogs.properties.target_resources.0.type": "Device", + "azure.auditlogs.result_signature": "None" } \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/sample_event.json b/packages/azure/data_stream/platformlogs/sample_event.json index 0f309e40e67..85752e6fa49 100644 --- a/packages/azure/data_stream/platformlogs/sample_event.json +++ b/packages/azure/data_stream/platformlogs/sample_event.json @@ -1,83 +1,77 @@ { - "_index": ".ds-logs-azure.platformlogs-default-000001", - "_type": "_doc", - "_id": "BHSwg3UBWgbgrXIaDOF-", - "_score": null, - "_source": { - "agent": { - "hostname": "DESKTOP-RFOOE09", - "name": "DESKTOP-RFOOE09", - "id": "c1118415-bcb7-4cf9-b64d-a6c6e8ebcfac", - "type": "filebeat", - "ephemeral_id": "d3c4d56c-e7c7-489e-9d25-683452d16ec9", - "version": "7.10.0" - }, - "elastic_agent": { - "id": "02f4e39d-8a1b-4506-a531-b45d0f492ee7", - "version": "7.10.0", - "snapshot": false - }, - "azure-eventhub": { - "sequence_number": 15, - "consumer_group": "$Default", - "offset": 4294976088, - "eventhub": "insights-logs-operationallogs", - "enqueued_time": "2020-11-05T14:08:28.137Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure", - "region": "West Europe" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-05T14:07:32.000Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.platformlogs" - }, - "host": { - "name": "DESKTOP-RFOOE09" - }, - "event": { - "ingested": "2020-11-01T12:02:34.237205200Z", - "kind": "event", - "action": "Retreive Namespace", - "dataset": "azure.platformlogs", - "outcome": "succeeded" - }, - "azure": { - "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", - "platformlogs": { - "Status": "Succeeded", + "agent": { + "hostname": "DESKTOP-RFOOE09", + "name": "DESKTOP-RFOOE09", + "id": "c1118415-bcb7-4cf9-b64d-a6c6e8ebcfac", + "type": "filebeat", + "ephemeral_id": "d3c4d56c-e7c7-489e-9d25-683452d16ec9", + "version": "7.10.0" + }, + "elastic_agent": { + "id": "02f4e39d-8a1b-4506-a531-b45d0f492ee7", + "version": "7.10.0", + "snapshot": false + }, + "azure-eventhub": { + "sequence_number": 15, + "consumer_group": "$Default", + "offset": 4294976088, + "eventhub": "insights-logs-operationallogs", + "enqueued_time": "2020-11-05T14:08:28.137Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure", + "region": "West Europe" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-05T14:07:32.000Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.platformlogs" + }, + "host": { + "name": "DESKTOP-RFOOE09" + }, + "event": { + "ingested": "2020-11-01T12:02:34.237205200Z", + "kind": "event", + "action": "Retreive Namespace", + "dataset": "azure.platformlogs", + "outcome": "succeeded" + }, + "azure": { + "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", + "platformlogs": { + "Status": "Succeeded", + "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "Caller": "Portal", + "ActivityId": "5890c6fc-fc6b-47cd-971a-2366a1641d99", + "EventTimeString": "11/5/2020 2:07:32 PM +00:00", + "Environment": "PROD", + "category": "OperationalLogs", + "event_category": "Administrative", + "ScaleUnit": "PROD-AM3-AZ501", + "properties": { "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", - "Caller": "Portal", - "ActivityId": "5890c6fc-fc6b-47cd-971a-2366a1641d99", - "EventTimeString": "11/5/2020 2:07:32 PM +00:00", - "Environment": "PROD", - "category": "OperationalLogs", - "event_category": "Administrative", - "ScaleUnit": "PROD-AM3-AZ501", - "properties": { - "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", - "TrackingId": "5890c6fc-fc6b-47cd-971a-2366a1641d99_M8CH3_M8CH3_G8S3", - "Namespace": "obstesteventhubs", - "Via": "https://obstesteventhubs.servicebus.windows.net/$Resources/eventhubs?api-version=2017-04\u0026$skip=0\u0026$top=100" - } - }, - "resource": { - "provider": "MICROSOFT.EVENTHUB/NAMESPACES", - "name": "OBSTESTEVENTHUBS", - "id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", - "group": "OBS-TEST" + "TrackingId": "5890c6fc-fc6b-47cd-971a-2366a1641d99_M8CH3_M8CH3_G8S3", + "Namespace": "obstesteventhubs", + "Via": "https://obstesteventhubs.servicebus.windows.net/$Resources/eventhubs?api-version=2017-04\u0026$skip=0\u0026$top=100" } + }, + "resource": { + "provider": "MICROSOFT.EVENTHUB/NAMESPACES", + "name": "OBSTESTEVENTHUBS", + "id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", + "group": "OBS-TEST" } } } \ No newline at end of file diff --git a/packages/azure/data_stream/signinlogs/sample_event.json b/packages/azure/data_stream/signinlogs/sample_event.json index fb4176e3c19..4aa0d3b2abb 100644 --- a/packages/azure/data_stream/signinlogs/sample_event.json +++ b/packages/azure/data_stream/signinlogs/sample_event.json @@ -1,77 +1,71 @@ { - "_index": ".ds-logs-azure.signinlogs-default-000001", - "_type": "_doc", - "_id": "bQlEe3UBm_qs2Y3aNZPq", - "_score": null, - "_source": { - "log": { - "level": "Information" - }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-signinlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-02T08:51:36.997Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.auditlogs" - }, - "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", - "azure.resource.provider": "Microsoft.aadiam", - "azure.signinlogs.category": "SignInLogs", - "azure.signinlogs.identity": "Test LTest", - "azure.signinlogs.operation_name": "Sign-in activity", - "azure.signinlogs.operation_version": "1.0", - "azure.signinlogs.properties.app_display_name": "Office 365", - "azure.signinlogs.properties.app_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.client_app_used": "Browser", - "azure.signinlogs.properties.conditional_access_status": "notApplied", - "azure.signinlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.created_at": "2019-10-18T04:45:48.0729893-05:00", - "azure.signinlogs.properties.device_detail.browser": "Chrome 77.0.3865", - "azure.signinlogs.properties.device_detail.device_id": "", - "azure.signinlogs.properties.device_detail.operating_system": "MacOs", - "azure.signinlogs.properties.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.ip_address": "81.171.241.231", - "azure.signinlogs.properties.is_interactive": false, - "azure.signinlogs.properties.original_request_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.processing_time_ms": 239, - "azure.signinlogs.properties.risk_detail": "none", - "azure.signinlogs.properties.risk_level_aggregated": "none", - "azure.signinlogs.properties.risk_level_during_signin": "none", - "azure.signinlogs.properties.risk_state": "none", - "azure.signinlogs.properties.service_principal_id": "", - "azure.signinlogs.properties.status.error_code": 50140, - "azure.signinlogs.properties.token_issuer_name": "", - "azure.signinlogs.properties.token_issuer_type": "AzureAD", - "azure.signinlogs.properties.user_display_name": "Test LTest", - "azure.signinlogs.properties.user_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.user_principal_name": "test@elastic.co", - "azure.signinlogs.result_description": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", - "azure.signinlogs.result_signature": "None", - "azure.signinlogs.result_type": "50140", - "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "cloud.provider": "azure", - "event.action": "Sign-in activity", - "event.category": [ - "authentication" - ] - } + "log": { + "level": "Information" + }, + "azure-eventhub": { + "sequence_number": 643, + "consumer_group": "$Default", + "offset": 107374182400, + "eventhub": "insights-signinlogs-logs", + "enqueued_time": "2020-11-02T08:59:38.905Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-02T08:51:36.997Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.auditlogs" + }, + "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", + "azure.resource.provider": "Microsoft.aadiam", + "azure.signinlogs.category": "SignInLogs", + "azure.signinlogs.identity": "Test LTest", + "azure.signinlogs.operation_name": "Sign-in activity", + "azure.signinlogs.operation_version": "1.0", + "azure.signinlogs.properties.app_display_name": "Office 365", + "azure.signinlogs.properties.app_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.client_app_used": "Browser", + "azure.signinlogs.properties.conditional_access_status": "notApplied", + "azure.signinlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.created_at": "2019-10-18T04:45:48.0729893-05:00", + "azure.signinlogs.properties.device_detail.browser": "Chrome 77.0.3865", + "azure.signinlogs.properties.device_detail.device_id": "", + "azure.signinlogs.properties.device_detail.operating_system": "MacOs", + "azure.signinlogs.properties.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.ip_address": "81.171.241.231", + "azure.signinlogs.properties.is_interactive": false, + "azure.signinlogs.properties.original_request_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.processing_time_ms": 239, + "azure.signinlogs.properties.risk_detail": "none", + "azure.signinlogs.properties.risk_level_aggregated": "none", + "azure.signinlogs.properties.risk_level_during_signin": "none", + "azure.signinlogs.properties.risk_state": "none", + "azure.signinlogs.properties.service_principal_id": "", + "azure.signinlogs.properties.status.error_code": 50140, + "azure.signinlogs.properties.token_issuer_name": "", + "azure.signinlogs.properties.token_issuer_type": "AzureAD", + "azure.signinlogs.properties.user_display_name": "Test LTest", + "azure.signinlogs.properties.user_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.user_principal_name": "test@elastic.co", + "azure.signinlogs.result_description": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", + "azure.signinlogs.result_signature": "None", + "azure.signinlogs.result_type": "50140", + "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "cloud.provider": "azure", + "event.action": "Sign-in activity", + "event.category": [ + "authentication" + ] } \ No newline at end of file diff --git a/packages/azure/docs/README.md b/packages/azure/docs/README.md index e68e15e9653..2e4949d1943 100644 --- a/packages/azure/docs/README.md +++ b/packages/azure/docs/README.md @@ -62,113 +62,107 @@ An example event for `activitylogs` looks as following: ```$json { - "_index": ".ds-logs-azure.activitylogs-default-000001", - "_type": "_doc", - "_id": "bQlEe3UBm_qs2Y3aNZPq", - "_score": null, - "_source": { - "log": { - "level": "Information" + "log": { + "level": "Information" + }, + "azure-eventhub": { + "sequence_number": 643, + "consumer_group": "$Default", + "offset": 107374182400, + "eventhub": "insights-activity-logs", + "enqueued_time": "2020-11-02T08:59:38.905Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-02T08:51:36.997Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.activitylogs" + }, + "event": { + "duration": "0", + "ingested": "2020-10-30T20:47:48.123859400Z", + "kind": "event", + "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", + "dataset": "azure.activitylogs", + "outcome": "success" + }, + "azure": { + "subscription_id": "3f041b6d-fc31-41d8-8ff6-e5f16e6747ff", + "resource": { + "provider": "MICROSOFT.RESOURCES/DEPLOYMENTS", + "name": "NOMARKETPLACE", + "id": "/SUBSCRIPTIONS/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.RESOURCES/DEPLOYMENTS/NOMARKETPLACE", + "group": "OBS-TEST" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-activity-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-02T08:51:36.997Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.activitylogs" - }, - "event": { - "duration": "0", - "ingested": "2020-10-30T20:47:48.123859400Z", - "kind": "event", - "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", - "dataset": "azure.activitylogs", - "outcome": "success" - }, - "azure": { - "subscription_id": "3f041b6d-fc31-41d8-8ff6-e5f16e6747ff", - "resource": { - "provider": "MICROSOFT.RESOURCES/DEPLOYMENTS", - "name": "NOMARKETPLACE", - "id": "/SUBSCRIPTIONS/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.RESOURCES/DEPLOYMENTS/NOMARKETPLACE", - "group": "OBS-TEST" - }, - "correlation_id": "876190b4-5b99-4a39-b725-4f5644911cf0", - "activitylogs": { - "operation_name": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", - "result_type": "Success", - "identity": { - "authorization": { - "evidence": { - "role_definition_id": "8e3af657a8ff443ca75c2fe8c4bcb635", - "role": "Owner", - "role_assignment_scope": "/providers/Microsoft.Management/managementGroups/5341238b-665c-4eb4-b259-b250371ae430", - "role_assignment_id": "7f06f09dd6764b44930adbec3f10e92b", - "principal_type": "User", - "principal_id": "68b1adf93eb744b08eb8ce96522a08d3" - }, - "scope": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace", - "action": "Microsoft.Resources/deployments/write" - }, - "claims": { - "xms_tcdt": "1469565974", - "aio": "ATQAy/8RAAAAsL67UQMOHZv3izTDRJfvJN5UyON9ktUszzPj08K8aURsbhxhR0niz9s1Pxm9U1lI", - "iss": "https://sts.windows.net/4fa94b7d-a743-486f-abcc-6c276c44cf4b/", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "a9L2WR3XZN5ANzAqwLx_4aamU49JG6kqaE5JZkXdeNs", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Doe", - "http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", - "http://schemas_microsoft_com/identity/claims/tenantid": "4fa94b7d-a743-486f-abcc-6c276c44cf4b", - "puid": "1003200045B17AD4", - "wids": "5d6b6bb7-de71-4623-b4af-96380a352509", - "http://schemas_microsoft_com/claims/authnclassreference": "1", - "exp": "1604310019", - "ipaddr": "77.170.179.229", - "iat": "1604306119", - "http://schemas_microsoft_com/identity/claims/objectidentifier": "68b1adf9-3eb7-44b0-8eb8-ce96522a08d3", - "http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", - "ver": "1.0", - "groups": "644c6686-9ef1-4b69-9410-107664a9e1f0,9ed1993c-ce9c-4915-a04d-58c6f5f7ee12", - "uti": "rqr63RW_Kk6ztuomENMQAA", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "john@gmail.com", - "aud": "https://management.core.windows.net/", - "nbf": "1604306119", - "appidacr": "2", - "rh": "0.AAAAfUupT0Onb0irzGwnbETPS4NAS8SwO8FJtH2XTlPL3zxRAA8.", - "appid": "c44b4083-3bb0-49c1-b47d-974e53cbdf3c", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", - "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": "john@gmail.com" + "correlation_id": "876190b4-5b99-4a39-b725-4f5644911cf0", + "activitylogs": { + "operation_name": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", + "result_type": "Success", + "identity": { + "authorization": { + "evidence": { + "role_definition_id": "8e3af657a8ff443ca75c2fe8c4bcb635", + "role": "Owner", + "role_assignment_scope": "/providers/Microsoft.Management/managementGroups/5341238b-665c-4eb4-b259-b250371ae430", + "role_assignment_id": "7f06f09dd6764b44930adbec3f10e92b", + "principal_type": "User", + "principal_id": "68b1adf93eb744b08eb8ce96522a08d3" }, - "claims_initiated_by_user": { - "schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims" - } + "scope": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace", + "action": "Microsoft.Resources/deployments/write" }, - "category": "Administrative", - "event_category": "Administrative", - "result_signature": "Succeeded.", - "properties": { - "eventCategory": "Administrative", - "hierarchy": "", - "message": "Microsoft.Resources/deployments/write", - "entity": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace" + "claims": { + "xms_tcdt": "1469565974", + "aio": "ATQAy/8RAAAAsL67UQMOHZv3izTDRJfvJN5UyON9ktUszzPj08K8aURsbhxhR0niz9s1Pxm9U1lI", + "iss": "https://sts.windows.net/4fa94b7d-a743-486f-abcc-6c276c44cf4b/", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "a9L2WR3XZN5ANzAqwLx_4aamU49JG6kqaE5JZkXdeNs", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Doe", + "http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", + "http://schemas_microsoft_com/identity/claims/tenantid": "4fa94b7d-a743-486f-abcc-6c276c44cf4b", + "puid": "1003200045B17AD4", + "wids": "5d6b6bb7-de71-4623-b4af-96380a352509", + "http://schemas_microsoft_com/claims/authnclassreference": "1", + "exp": "1604310019", + "ipaddr": "77.170.179.229", + "iat": "1604306119", + "http://schemas_microsoft_com/identity/claims/objectidentifier": "68b1adf9-3eb7-44b0-8eb8-ce96522a08d3", + "http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", + "ver": "1.0", + "groups": "644c6686-9ef1-4b69-9410-107664a9e1f0,9ed1993c-ce9c-4915-a04d-58c6f5f7ee12", + "uti": "rqr63RW_Kk6ztuomENMQAA", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "john@gmail.com", + "aud": "https://management.core.windows.net/", + "nbf": "1604306119", + "appidacr": "2", + "rh": "0.AAAAfUupT0Onb0irzGwnbETPS4NAS8SwO8FJtH2XTlPL3zxRAA8.", + "appid": "c44b4083-3bb0-49c1-b47d-974e53cbdf3c", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", + "http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": "john@gmail.com" + }, + "claims_initiated_by_user": { + "schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims" } + }, + "category": "Administrative", + "event_category": "Administrative", + "result_signature": "Succeeded.", + "properties": { + "eventCategory": "Administrative", + "hierarchy": "", + "message": "Microsoft.Resources/deployments/write", + "entity": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace" } } } @@ -303,85 +297,79 @@ An example event for `platformlogs` looks as following: ```$json { - "_index": ".ds-logs-azure.platformlogs-default-000001", - "_type": "_doc", - "_id": "BHSwg3UBWgbgrXIaDOF-", - "_score": null, - "_source": { - "agent": { - "hostname": "DESKTOP-RFOOE09", - "name": "DESKTOP-RFOOE09", - "id": "c1118415-bcb7-4cf9-b64d-a6c6e8ebcfac", - "type": "filebeat", - "ephemeral_id": "d3c4d56c-e7c7-489e-9d25-683452d16ec9", - "version": "7.10.0" - }, - "elastic_agent": { - "id": "02f4e39d-8a1b-4506-a531-b45d0f492ee7", - "version": "7.10.0", - "snapshot": false - }, - "azure-eventhub": { - "sequence_number": 15, - "consumer_group": "$Default", - "offset": 4294976088, - "eventhub": "insights-logs-operationallogs", - "enqueued_time": "2020-11-05T14:08:28.137Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure", - "region": "West Europe" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-05T14:07:32.000Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.platformlogs" - }, - "host": { - "name": "DESKTOP-RFOOE09" - }, - "event": { - "ingested": "2020-11-01T12:02:34.237205200Z", - "kind": "event", - "action": "Retreive Namespace", - "dataset": "azure.platformlogs", - "outcome": "succeeded" - }, - "azure": { - "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", - "platformlogs": { - "Status": "Succeeded", + "agent": { + "hostname": "DESKTOP-RFOOE09", + "name": "DESKTOP-RFOOE09", + "id": "c1118415-bcb7-4cf9-b64d-a6c6e8ebcfac", + "type": "filebeat", + "ephemeral_id": "d3c4d56c-e7c7-489e-9d25-683452d16ec9", + "version": "7.10.0" + }, + "elastic_agent": { + "id": "02f4e39d-8a1b-4506-a531-b45d0f492ee7", + "version": "7.10.0", + "snapshot": false + }, + "azure-eventhub": { + "sequence_number": 15, + "consumer_group": "$Default", + "offset": 4294976088, + "eventhub": "insights-logs-operationallogs", + "enqueued_time": "2020-11-05T14:08:28.137Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure", + "region": "West Europe" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-05T14:07:32.000Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.platformlogs" + }, + "host": { + "name": "DESKTOP-RFOOE09" + }, + "event": { + "ingested": "2020-11-01T12:02:34.237205200Z", + "kind": "event", + "action": "Retreive Namespace", + "dataset": "azure.platformlogs", + "outcome": "succeeded" + }, + "azure": { + "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", + "platformlogs": { + "Status": "Succeeded", + "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "Caller": "Portal", + "ActivityId": "5890c6fc-fc6b-47cd-971a-2366a1641d99", + "EventTimeString": "11/5/2020 2:07:32 PM +00:00", + "Environment": "PROD", + "category": "OperationalLogs", + "event_category": "Administrative", + "ScaleUnit": "PROD-AM3-AZ501", + "properties": { "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", - "Caller": "Portal", - "ActivityId": "5890c6fc-fc6b-47cd-971a-2366a1641d99", - "EventTimeString": "11/5/2020 2:07:32 PM +00:00", - "Environment": "PROD", - "category": "OperationalLogs", - "event_category": "Administrative", - "ScaleUnit": "PROD-AM3-AZ501", - "properties": { - "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", - "TrackingId": "5890c6fc-fc6b-47cd-971a-2366a1641d99_M8CH3_M8CH3_G8S3", - "Namespace": "obstesteventhubs", - "Via": "https://obstesteventhubs.servicebus.windows.net/$Resources/eventhubs?api-version=2017-04\u0026$skip=0\u0026$top=100" - } - }, - "resource": { - "provider": "MICROSOFT.EVENTHUB/NAMESPACES", - "name": "OBSTESTEVENTHUBS", - "id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", - "group": "OBS-TEST" + "TrackingId": "5890c6fc-fc6b-47cd-971a-2366a1641d99_M8CH3_M8CH3_G8S3", + "Namespace": "obstesteventhubs", + "Via": "https://obstesteventhubs.servicebus.windows.net/$Resources/eventhubs?api-version=2017-04\u0026$skip=0\u0026$top=100" } + }, + "resource": { + "provider": "MICROSOFT.EVENTHUB/NAMESPACES", + "name": "OBSTESTEVENTHUBS", + "id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", + "group": "OBS-TEST" } } } @@ -507,75 +495,69 @@ An example event for `auditlogs` looks as following: ```$json { - "_index": ".ds-logs-azure.auditlogs-default-000001", - "_type": "_doc", - "_id": "bQlEe3UBm_qs2Y3aNZPq", - "_score": null, - "_source": { - "log": { - "level": "Information" - }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-auditlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-02T08:51:36.997Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.auditlogs" - }, - "event": { - "duration": "0", - "ingested": "2020-10-30T20:47:48.123859400Z", - "kind": "event", - "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", - "dataset": "azure.auditlogs", - "outcome": "success" - }, - "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", - "azure.resource.provider": "Microsoft.aadiam", - "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.category": "AuditLogs", - "azure.auditlogs.identity": "Device Registration Service", - "azure.auditlogs.operation_name": "Update device", - "azure.auditlogs.operation_version": "1.0", - "azure.auditlogs.properties.activity_datetime": "2019-10-18T15:30:51.0273716+00:00", - "azure.auditlogs.properties.activity_display_name": "Update device", - "azure.auditlogs.properties.category": "Device", - "azure.auditlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.id": "Directory_ESQ", - "azure.auditlogs.properties.initiated_by.app.appId": null, - "azure.auditlogs.properties.initiated_by.app.displayName": "Device Registration Service", - "azure.auditlogs.properties.initiated_by.app.servicePrincipalId": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.initiated_by.app.servicePrincipalName": null, - "azure.auditlogs.properties.logged_by_service": "Core Directory", - "azure.auditlogs.properties.operation_type": "Update", - "azure.auditlogs.properties.result_reason": "", - "azure.auditlogs.properties.target_resources.0.display_name": "LAPTOP-12", - "azure.auditlogs.properties.target_resources.0.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.display_name": "Included Updated Properties", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "\"\"", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": null, - "azure.auditlogs.properties.target_resources.0.type": "Device", - "azure.auditlogs.result_signature": "None" - } + "log": { + "level": "Information" + }, + "azure-eventhub": { + "sequence_number": 643, + "consumer_group": "$Default", + "offset": 107374182400, + "eventhub": "insights-auditlogs-logs", + "enqueued_time": "2020-11-02T08:59:38.905Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-02T08:51:36.997Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.auditlogs" + }, + "event": { + "duration": "0", + "ingested": "2020-10-30T20:47:48.123859400Z", + "kind": "event", + "action": "MICROSOFT.RESOURCES/DEPLOYMENTS/WRITE", + "dataset": "azure.auditlogs", + "outcome": "success" + }, + "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", + "azure.resource.provider": "Microsoft.aadiam", + "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.category": "AuditLogs", + "azure.auditlogs.identity": "Device Registration Service", + "azure.auditlogs.operation_name": "Update device", + "azure.auditlogs.operation_version": "1.0", + "azure.auditlogs.properties.activity_datetime": "2019-10-18T15:30:51.0273716+00:00", + "azure.auditlogs.properties.activity_display_name": "Update device", + "azure.auditlogs.properties.category": "Device", + "azure.auditlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.properties.id": "Directory_ESQ", + "azure.auditlogs.properties.initiated_by.app.appId": null, + "azure.auditlogs.properties.initiated_by.app.displayName": "Device Registration Service", + "azure.auditlogs.properties.initiated_by.app.servicePrincipalId": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.properties.initiated_by.app.servicePrincipalName": null, + "azure.auditlogs.properties.logged_by_service": "Core Directory", + "azure.auditlogs.properties.operation_type": "Update", + "azure.auditlogs.properties.result_reason": "", + "azure.auditlogs.properties.target_resources.0.display_name": "LAPTOP-12", + "azure.auditlogs.properties.target_resources.0.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.auditlogs.properties.target_resources.0.modified_properties.0.display_name": "Included Updated Properties", + "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "\"\"", + "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": null, + "azure.auditlogs.properties.target_resources.0.type": "Device", + "azure.auditlogs.result_signature": "None" } ``` @@ -716,81 +698,75 @@ An example event for `signinlogs` looks as following: ```$json { - "_index": ".ds-logs-azure.signinlogs-default-000001", - "_type": "_doc", - "_id": "bQlEe3UBm_qs2Y3aNZPq", - "_score": null, - "_source": { - "log": { - "level": "Information" - }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-signinlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], - "cloud": { - "provider": "azure" - }, - "input": { - "type": "azure-eventhub" - }, - "@timestamp": "2020-11-02T08:51:36.997Z", - "ecs": { - "version": "1.5.0" - }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "azure.auditlogs" - }, - "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", - "azure.resource.provider": "Microsoft.aadiam", - "azure.signinlogs.category": "SignInLogs", - "azure.signinlogs.identity": "Test LTest", - "azure.signinlogs.operation_name": "Sign-in activity", - "azure.signinlogs.operation_version": "1.0", - "azure.signinlogs.properties.app_display_name": "Office 365", - "azure.signinlogs.properties.app_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.client_app_used": "Browser", - "azure.signinlogs.properties.conditional_access_status": "notApplied", - "azure.signinlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.created_at": "2019-10-18T04:45:48.0729893-05:00", - "azure.signinlogs.properties.device_detail.browser": "Chrome 77.0.3865", - "azure.signinlogs.properties.device_detail.device_id": "", - "azure.signinlogs.properties.device_detail.operating_system": "MacOs", - "azure.signinlogs.properties.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.ip_address": "81.171.241.231", - "azure.signinlogs.properties.is_interactive": false, - "azure.signinlogs.properties.original_request_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.processing_time_ms": 239, - "azure.signinlogs.properties.risk_detail": "none", - "azure.signinlogs.properties.risk_level_aggregated": "none", - "azure.signinlogs.properties.risk_level_during_signin": "none", - "azure.signinlogs.properties.risk_state": "none", - "azure.signinlogs.properties.service_principal_id": "", - "azure.signinlogs.properties.status.error_code": 50140, - "azure.signinlogs.properties.token_issuer_name": "", - "azure.signinlogs.properties.token_issuer_type": "AzureAD", - "azure.signinlogs.properties.user_display_name": "Test LTest", - "azure.signinlogs.properties.user_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.signinlogs.properties.user_principal_name": "test@elastic.co", - "azure.signinlogs.result_description": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", - "azure.signinlogs.result_signature": "None", - "azure.signinlogs.result_type": "50140", - "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "cloud.provider": "azure", - "event.action": "Sign-in activity", - "event.category": [ - "authentication" - ] - } + "log": { + "level": "Information" + }, + "azure-eventhub": { + "sequence_number": 643, + "consumer_group": "$Default", + "offset": 107374182400, + "eventhub": "insights-signinlogs-logs", + "enqueued_time": "2020-11-02T08:59:38.905Z" + }, + "tags": [ + "forwarded" + ], + "cloud": { + "provider": "azure" + }, + "input": { + "type": "azure-eventhub" + }, + "@timestamp": "2020-11-02T08:51:36.997Z", + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "azure.auditlogs" + }, + "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", + "azure.resource.provider": "Microsoft.aadiam", + "azure.signinlogs.category": "SignInLogs", + "azure.signinlogs.identity": "Test LTest", + "azure.signinlogs.operation_name": "Sign-in activity", + "azure.signinlogs.operation_version": "1.0", + "azure.signinlogs.properties.app_display_name": "Office 365", + "azure.signinlogs.properties.app_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.client_app_used": "Browser", + "azure.signinlogs.properties.conditional_access_status": "notApplied", + "azure.signinlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.created_at": "2019-10-18T04:45:48.0729893-05:00", + "azure.signinlogs.properties.device_detail.browser": "Chrome 77.0.3865", + "azure.signinlogs.properties.device_detail.device_id": "", + "azure.signinlogs.properties.device_detail.operating_system": "MacOs", + "azure.signinlogs.properties.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.ip_address": "81.171.241.231", + "azure.signinlogs.properties.is_interactive": false, + "azure.signinlogs.properties.original_request_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.processing_time_ms": 239, + "azure.signinlogs.properties.risk_detail": "none", + "azure.signinlogs.properties.risk_level_aggregated": "none", + "azure.signinlogs.properties.risk_level_during_signin": "none", + "azure.signinlogs.properties.risk_state": "none", + "azure.signinlogs.properties.service_principal_id": "", + "azure.signinlogs.properties.status.error_code": 50140, + "azure.signinlogs.properties.token_issuer_name": "", + "azure.signinlogs.properties.token_issuer_type": "AzureAD", + "azure.signinlogs.properties.user_display_name": "Test LTest", + "azure.signinlogs.properties.user_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.signinlogs.properties.user_principal_name": "test@elastic.co", + "azure.signinlogs.result_description": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", + "azure.signinlogs.result_signature": "None", + "azure.signinlogs.result_type": "50140", + "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "cloud.provider": "azure", + "event.action": "Sign-in activity", + "event.category": [ + "authentication" + ] } ``` diff --git a/packages/kubernetes/data_stream/apiserver/sample_event.json b/packages/kubernetes/data_stream/apiserver/sample_event.json index d1491ff9583..e17aa1b2baf 100644 --- a/packages/kubernetes/data_stream/apiserver/sample_event.json +++ b/packages/kubernetes/data_stream/apiserver/sample_event.json @@ -1,86 +1,72 @@ { - "_index": ".ds-metrics-kubernetes.apiserver-default-000001", - "_id": "XVh163IBolOt49UrV2yq", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:30:34.616Z", - "metricset": { - "name": "apiserver", - "period": 30000 - }, - "service": { - "address": "10.96.0.1:443", - "type": "kubernetes" - }, - "event": { - "dataset": "kubernetes.apiserver", - "module": "kubernetes", - "duration": 114780772 - }, - "kubernetes": { - "apiserver": { - "request": { - "client": "metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format", - "version": "v1", - "count": 3, - "scope": "cluster", - "content_type": "application/vnd.kubernetes.protobuf", - "code": "200", - "verb": "LIST", - "component": "apiserver", - "resource": "nodes" - } - } - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.apiserver" - }, - "stream": { - "dataset": "kubernetes.apiserver", - "namespace": "default", - "type": "metrics" - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" + "@timestamp": "2020-06-25T12:30:34.616Z", + "metricset": { + "name": "apiserver", + "period": 30000 + }, + "service": { + "address": "10.96.0.1:443", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.apiserver", + "module": "kubernetes", + "duration": 114780772 + }, + "kubernetes": { + "apiserver": { + "request": { + "client": "metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format", + "version": "v1", + "count": 3, + "scope": "cluster", + "content_type": "application/vnd.kubernetes.protobuf", + "code": "200", + "verb": "LIST", + "component": "apiserver", + "resource": "nodes" } } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:30:34.616Z" - ] + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1593088234616 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.apiserver" + }, + "stream": { + "dataset": "kubernetes.apiserver", + "namespace": "default", + "type": "metrics" + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/container/sample_event.json b/packages/kubernetes/data_stream/container/sample_event.json index d3ec8a444b7..1ae43a4470f 100644 --- a/packages/kubernetes/data_stream/container/sample_event.json +++ b/packages/kubernetes/data_stream/container/sample_event.json @@ -1,177 +1,160 @@ { - "_index": ".ds-metrics-kubernetes.container-default-000001", - "_id": "y1h363IBolOt49UrGcjO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:32:29.748Z", - "kubernetes": { - "namespace": "kube-system", - "node": { - "name": "minikube" + "@timestamp": "2020-06-25T12:32:29.748Z", + "kubernetes": { + "namespace": "kube-system", + "node": { + "name": "minikube" + }, + "pod": { + "name": "metricbeat-g9fc6" + }, + "container": { + "rootfs": { + "used": { + "bytes": 61440 + }, + "inodes": { + "used": 17 + }, + "available": { + "bytes": 6724222976 + }, + "capacity": { + "bytes": 17361141760 + } }, - "pod": { - "name": "metricbeat-g9fc6" + "logs": { + "used": { + "bytes": 1617920 + }, + "inodes": { + "count": 9768928, + "used": 223910, + "free": 9545018 + }, + "available": { + "bytes": 6724222976 + }, + "capacity": { + "bytes": 17361141760 + } }, - "container": { - "rootfs": { - "used": { - "bytes": 61440 - }, - "inodes": { - "used": 17 + "start_time": "2020-06-25T07:19:37Z", + "name": "metricbeat", + "cpu": { + "usage": { + "node": { + "pct": 0.00015289625 }, - "available": { - "bytes": 6724222976 + "limit": { + "pct": 0.00015289625 }, - "capacity": { - "bytes": 17361141760 + "nanocores": 611585, + "core": { + "ns": 12206519774 } + } + }, + "memory": { + "pagefaults": 10164, + "majorpagefaults": 528, + "available": { + "bytes": 188600320 }, - "logs": { - "used": { - "bytes": 1617920 - }, - "inodes": { - "count": 9768928, - "used": 223910, - "free": 9545018 + "usage": { + "limit": { + "pct": 0.005608354460473573 }, - "available": { - "bytes": 6724222976 - }, - "capacity": { - "bytes": 17361141760 + "bytes": 94306304, + "node": { + "pct": 0.005608354460473573 } }, - "start_time": "2020-06-25T07:19:37Z", - "name": "metricbeat", - "cpu": { - "usage": { - "node": { - "pct": 0.00015289625 - }, - "limit": { - "pct": 0.00015289625 - }, - "nanocores": 611585, - "core": { - "ns": 12206519774 - } - } + "workingset": { + "bytes": 21114880 }, - "memory": { - "pagefaults": 10164, - "majorpagefaults": 528, - "available": { - "bytes": 188600320 - }, - "usage": { - "limit": { - "pct": 0.005608354460473573 - }, - "bytes": 94306304, - "node": { - "pct": 0.005608354460473573 - } - }, - "workingset": { - "bytes": 21114880 - }, - "rss": { - "bytes": 18386944 - } + "rss": { + "bytes": 18386944 } } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.container" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.container" - }, - "host": { - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" - }, - "name": "minikube", - "id": "b0e83d397c054b8a99a431072fe4617b" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube" - }, - "metricset": { - "period": 10000, - "name": "container" - }, - "service": { - "address": "minikube:10250", - "type": "kubernetes" - }, - "event": { - "dataset": "kubernetes.container", - "module": "kubernetes", - "duration": 11091346 - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:32:29.748Z" + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.container" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.container" + }, + "host": { + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" ], - "kubernetes.container.start_time": [ - "2020-06-25T07:19:37.000Z" - ] + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + }, + "name": "minikube", + "id": "b0e83d397c054b8a99a431072fe4617b" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + }, + "metricset": { + "period": 10000, + "name": "container" + }, + "service": { + "address": "minikube:10250", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.container", + "module": "kubernetes", + "duration": 11091346 }, - "sort": [ - 1593088349748 - ] + "ecs": { + "version": "1.5.0" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/controllermanager/sample_event.json b/packages/kubernetes/data_stream/controllermanager/sample_event.json index 82223308f60..9d1b4207a45 100644 --- a/packages/kubernetes/data_stream/controllermanager/sample_event.json +++ b/packages/kubernetes/data_stream/controllermanager/sample_event.json @@ -1,117 +1,103 @@ { - "_index": ".ds-metrics-kubernetes.controllermanager-default-000001", - "_id": "qFh463IBolOt49UrBPYP", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:33:29.643Z", - "kubernetes": { - "controllermanager": { - "workqueue": { - "unfinished": { - "sec": 0 - }, - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - } + "@timestamp": "2020-06-25T12:33:29.643Z", + "kubernetes": { + "controllermanager": { + "workqueue": { + "unfinished": { + "sec": 0 }, - "name": "certificate" - } - }, - "event": { - "dataset": "kubernetes.controllermanager", - "module": "kubernetes", - "duration": 8893806 - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.controllermanager" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + } }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "name": "minikube" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube", - "type": "metricbeat" - }, - "metricset": { - "period": 10000, - "name": "controllermanager" - }, - "service": { - "address": "localhost:10252", - "type": "kubernetes" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.controllermanager" + "name": "certificate" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:33:29.643Z" - ] + "event": { + "dataset": "kubernetes.controllermanager", + "module": "kubernetes", + "duration": 8893806 + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.controllermanager" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "name": "minikube" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat" + }, + "metricset": { + "period": 10000, + "name": "controllermanager" + }, + "service": { + "address": "localhost:10252", + "type": "kubernetes" }, - "sort": [ - 1593088409643 - ] + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.controllermanager" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/event/sample_event.json b/packages/kubernetes/data_stream/event/sample_event.json index c03d90a126c..8226317aa9c 100644 --- a/packages/kubernetes/data_stream/event/sample_event.json +++ b/packages/kubernetes/data_stream/event/sample_event.json @@ -1,111 +1,88 @@ { - "_index": ".ds-metrics-kubernetes.event-default-000001", - "_id": "EVh163IBolOt49UrPGji", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:30:27.575Z", - "metricset": { - "name": "event" - }, - "stream": { - "dataset": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc" - }, - "ecs": { - "version": "1.5.0" - }, + "@timestamp": "2020-06-25T12:30:27.575Z", + "metricset": { + "name": "event" + }, + "stream": { + "dataset": "kubernetes.event", + "namespace": "default", + "type": "metrics" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "kubernetes.event", + "module": "kubernetes" + }, + "service": { + "type": "kubernetes" + }, + "kubernetes": { "event": { - "dataset": "kubernetes.event", - "module": "kubernetes" - }, - "service": { - "type": "kubernetes" - }, - "kubernetes": { - "event": { - "metadata": { - "uid": "604e39e0-862f-4615-9cec-8cb62299dea3", - "resource_version": "485630", - "timestamp": { - "created": "2020-06-25T07:20:25.000Z" - }, - "name": "monitor.161bb862545e3099", - "namespace": "beats", - "self_link": "/api/v1/namespaces/beats/events/monitor.161bb862545e3099", - "generate_name": "" - }, + "metadata": { + "uid": "604e39e0-862f-4615-9cec-8cb62299dea3", + "resource_version": "485630", "timestamp": { - "first_occurrence": "2020-06-25T07:20:25.000Z", - "last_occurrence": "2020-06-25T12:30:27.000Z" + "created": "2020-06-25T07:20:25.000Z" }, - "message": "Failed to find referenced backend beats/monitor: Elasticsearch.elasticsearch.k8s.elastic.co \"monitor\" not found", - "reason": "AssociationError", - "type": "Warning", - "count": 1861, - "source": { - "host": "", - "component": "kibana-association-controller" - }, - "involved_object": { - "api_version": "kibana.k8s.elastic.co/v1", - "resource_version": "101842", - "name": "monitor", - "kind": "Kibana", - "uid": "45a19de5-5eef-4090-a2d3-dbceb0a28af8" - } - } - }, - "dataset": { - "name": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" + "name": "monitor.161bb862545e3099", + "namespace": "beats", + "self_link": "/api/v1/namespaces/beats/events/monitor.161bb862545e3099", + "generate_name": "" + }, + "timestamp": { + "first_occurrence": "2020-06-25T07:20:25.000Z", + "last_occurrence": "2020-06-25T12:30:27.000Z" + }, + "message": "Failed to find referenced backend beats/monitor: Elasticsearch.elasticsearch.k8s.elastic.co \"monitor\" not found", + "reason": "AssociationError", + "type": "Warning", + "count": 1861, + "source": { + "host": "", + "component": "kibana-association-controller" + }, + "involved_object": { + "api_version": "kibana.k8s.elastic.co/v1", + "resource_version": "101842", + "name": "monitor", + "kind": "Kibana", + "uid": "45a19de5-5eef-4090-a2d3-dbceb0a28af8" } } }, - "fields": { - "kubernetes.event.timestamp.first_occurrence": [ - "2020-06-25T07:20:25.000Z" - ], - "kubernetes.event.timestamp.last_occurrence": [ - "2020-06-25T12:30:27.000Z" + "dataset": { + "name": "kubernetes.event", + "namespace": "default", + "type": "metrics" + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.event.metadata.timestamp.created": [ - "2020-06-25T07:20:25.000Z" + "mac": [ + "02:42:ac:11:00:0b" ], - "@timestamp": [ - "2020-06-25T12:30:27.575Z" - ] - }, - "sort": [ - 1593088227575 - ] + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/node/sample_event.json b/packages/kubernetes/data_stream/node/sample_event.json index a7df088af0a..1ef8b7d994b 100644 --- a/packages/kubernetes/data_stream/node/sample_event.json +++ b/packages/kubernetes/data_stream/node/sample_event.json @@ -1,175 +1,158 @@ { - "_index": ".ds-metrics-kubernetes.node-default-000001", - "_id": "Gll563IBolOt49UrFS2Q", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:34:39.723Z", - "event": { - "dataset": "kubernetes.node", - "module": "kubernetes", - "duration": 13042307 + "@timestamp": "2020-06-25T12:34:39.723Z", + "event": { + "dataset": "kubernetes.node", + "module": "kubernetes", + "duration": 13042307 + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "host": { + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" + "id": "b0e83d397c054b8a99a431072fe4617b" + }, + "metricset": { + "name": "node", + "period": 10000 + }, + "kubernetes": { + "labels": { + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "amd64", + "kubernetes_io/hostname": "minikube", + "kubernetes_io/os": "linux", + "node-role_kubernetes_io/master": "", + "beta_kubernetes_io/arch": "amd64" }, - "host": { - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "name": "minikube", - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" + "node": { + "memory": { + "available": { + "bytes": 12746428416 + }, + "usage": { + "bytes": 5670916096 + }, + "workingset": { + "bytes": 4068896768 + }, + "rss": { + "bytes": 3252125696 + }, + "pagefaults": 31680, + "majorpagefaults": 0 }, - "id": "b0e83d397c054b8a99a431072fe4617b" - }, - "metricset": { - "name": "node", - "period": 10000 - }, - "kubernetes": { - "labels": { - "beta_kubernetes_io/os": "linux", - "kubernetes_io/arch": "amd64", - "kubernetes_io/hostname": "minikube", - "kubernetes_io/os": "linux", - "node-role_kubernetes_io/master": "", - "beta_kubernetes_io/arch": "amd64" + "network": { + "rx": { + "bytes": 107077476, + "errors": 0 + }, + "tx": { + "bytes": 67457933, + "errors": 0 + } }, - "node": { - "memory": { - "available": { - "bytes": 12746428416 - }, - "usage": { - "bytes": 5670916096 - }, - "workingset": { - "bytes": 4068896768 - }, - "rss": { - "bytes": 3252125696 - }, - "pagefaults": 31680, - "majorpagefaults": 0 + "fs": { + "available": { + "bytes": 6655090688 }, - "network": { - "rx": { - "bytes": 107077476, - "errors": 0 - }, - "tx": { - "bytes": 67457933, - "errors": 0 - } + "capacity": { + "bytes": 17361141760 }, - "fs": { - "available": { - "bytes": 6655090688 - }, + "used": { + "bytes": 9689358336 + }, + "inodes": { + "count": 9768928, + "used": 224151, + "free": 9544777 + } + }, + "runtime": { + "imagefs": { "capacity": { "bytes": 17361141760 }, "used": { - "bytes": 9689358336 + "bytes": 8719928568 }, - "inodes": { - "count": 9768928, - "used": 224151, - "free": 9544777 - } - }, - "runtime": { - "imagefs": { - "capacity": { - "bytes": 17361141760 - }, - "used": { - "bytes": 8719928568 - }, - "available": { - "bytes": 6655090688 - } - } - }, - "start_time": "2020-06-25T07:18:38Z", - "name": "minikube", - "cpu": { - "usage": { - "core": { - "ns": 6136184971873 - }, - "nanocores": 455263291 + "available": { + "bytes": 6655090688 } } - } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.node" - }, - "agent": { + }, + "start_time": "2020-06-25T07:18:38Z", "name": "minikube", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" - }, - "ecs": { - "version": "1.5.0" + "cpu": { + "usage": { + "core": { + "ns": 6136184971873 + }, + "nanocores": 455263291 + } + } } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:34:39.723Z" - ], - "kubernetes.node.start_time": [ - "2020-06-25T07:18:38.000Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.node" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.node" + }, + "agent": { + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" }, - "sort": [ - 1593088479723 - ] + "ecs": { + "version": "1.5.0" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/pod/sample_event.json b/packages/kubernetes/data_stream/pod/sample_event.json index 1ead86b4ddd..857b5e83245 100644 --- a/packages/kubernetes/data_stream/pod/sample_event.json +++ b/packages/kubernetes/data_stream/pod/sample_event.json @@ -1,152 +1,135 @@ { - "_index": ".ds-metrics-kubernetes.pod-default-000001", - "_id": "4Vl563IBolOt49UrYz6x", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:34:59.729Z", - "kubernetes": { - "pod": { - "memory": { - "rss": { - "bytes": 7823360 - }, - "page_faults": 5742, - "major_page_faults": 0, - "usage": { - "limit": { - "pct": 0.0008033509820466402 - }, - "bytes": 13508608, - "node": { - "pct": 0.0008033509820466402 - } - }, - "available": { - "bytes": 0 - }, - "working_set": { - "bytes": 8556544 - } + "@timestamp": "2020-06-25T12:34:59.729Z", + "kubernetes": { + "pod": { + "memory": { + "rss": { + "bytes": 7823360 }, - "network": { - "rx": { - "bytes": 25671624, - "errors": 0 + "page_faults": 5742, + "major_page_faults": 0, + "usage": { + "limit": { + "pct": 0.0008033509820466402 }, - "tx": { - "errors": 0, - "bytes": 1092900259 + "bytes": 13508608, + "node": { + "pct": 0.0008033509820466402 } }, - "start_time": "2020-06-18T11:12:58Z", - "name": "kube-state-metrics-57cd6fdf9-hd959", - "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", - "cpu": { - "usage": { - "nanocores": 2811918, - "node": { - "pct": 0.0007029795 - }, - "limit": { - "pct": 0.0007029795 - } - } + "available": { + "bytes": 0 + }, + "working_set": { + "bytes": 8556544 } }, - "namespace": "kube-system", - "node": { - "name": "minikube" - } - }, - "event": { - "duration": 20735189, - "dataset": "kubernetes.pod", - "module": "kubernetes" - }, - "stream": { - "dataset": "kubernetes.pod", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "period": 10000, - "name": "pod" - }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.pod", - "namespace": "default" - }, - "host": { - "name": "minikube", - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" + "network": { + "rx": { + "bytes": 25671624, + "errors": 0 + }, + "tx": { + "errors": 0, + "bytes": 1092900259 + } }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ] + "start_time": "2020-06-18T11:12:58Z", + "name": "kube-state-metrics-57cd6fdf9-hd959", + "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", + "cpu": { + "usage": { + "nanocores": 2811918, + "node": { + "pct": 0.0007029795 + }, + "limit": { + "pct": 0.0007029795 + } + } + } }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "namespace": "kube-system", + "node": { "name": "minikube" } }, - "fields": { - "kubernetes.pod.start_time": [ - "2020-06-18T11:12:58.000Z" + "event": { + "duration": 20735189, + "dataset": "kubernetes.pod", + "module": "kubernetes" + }, + "stream": { + "dataset": "kubernetes.pod", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "period": 10000, + "name": "pod" + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "dataset": { + "type": "metrics", + "name": "kubernetes.pod", + "namespace": "default" + }, + "host": { + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" ], - "@timestamp": [ - "2020-06-25T12:34:59.729Z" + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" ] }, - "sort": [ - 1593088499729 - ] + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/proxy/sample_event.json b/packages/kubernetes/data_stream/proxy/sample_event.json index 9279877b692..30074f2136a 100644 --- a/packages/kubernetes/data_stream/proxy/sample_event.json +++ b/packages/kubernetes/data_stream/proxy/sample_event.json @@ -1,240 +1,226 @@ { - "_index": ".ds-metrics-kubernetes.proxy-default-000001", - "_id": "Z1l563IBolOt49Ur2FXO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:35:29.639Z", - "agent": { - "name": "minikube", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.proxy" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.proxy", - "namespace": "default" - }, - "host": { - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "name": "minikube", - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false + "@timestamp": "2020-06-25T12:35:29.639Z", + "agent": { + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.proxy" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.proxy", + "namespace": "default" + }, + "host": { + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "name": "minikube", + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" }, - "kubernetes": { - "proxy": { - "sync": { - "rules": { - "duration": { - "us": { - "sum": 763620.9329999998, - "count": 18, - "bucket": { - "1000": 0, - "2000": 0, - "4000": 0, - "8000": 0, - "16000": 0, - "32000": 10, - "64000": 16, - "128000": 17, - "256000": 18, - "512000": 18, - "1024000": 18, - "2048000": 18, - "4096000": 18, - "8192000": 18, - "16384000": 18, - "+Inf": 18 - } - } - } - }, - "networkprogramming": { - "duration": { - "us": { - "count": 19, - "bucket": { - "0": 0, - "250000": 4, - "500000": 8, - "1000000": 11, - "2000000": 11, - "3000000": 11, - "4000000": 11, - "5000000": 11, - "6000000": 11, - "7000000": 11, - "8000000": 11, - "9000000": 11, - "10000000": 11, - "11000000": 11, - "12000000": 11, - "13000000": 11, - "14000000": 11, - "15000000": 11, - "16000000": 11, - "17000000": 11, - "18000000": 11, - "19000000": 11, - "20000000": 11, - "21000000": 11, - "22000000": 11, - "23000000": 11, - "24000000": 11, - "25000000": 11, - "26000000": 11, - "27000000": 11, - "28000000": 11, - "29000000": 11, - "30000000": 11, - "31000000": 11, - "32000000": 11, - "33000000": 11, - "34000000": 11, - "35000000": 11, - "36000000": 11, - "37000000": 11, - "38000000": 11, - "39000000": 11, - "40000000": 11, - "41000000": 11, - "42000000": 11, - "43000000": 11, - "44000000": 11, - "45000000": 11, - "46000000": 11, - "47000000": 11, - "48000000": 11, - "49000000": 11, - "50000000": 11, - "51000000": 11, - "52000000": 11, - "53000000": 11, - "54000000": 11, - "55000000": 11, - "56000000": 11, - "57000000": 11, - "58000000": 11, - "59000000": 11, - "60000000": 11, - "65000000": 11, - "70000000": 11, - "75000000": 11, - "80000000": 11, - "85000000": 11, - "90000000": 11, - "95000000": 11, - "100000000": 11, - "105000000": 11, - "110000000": 11, - "115000000": 11, - "120000000": 11, - "150000000": 11, - "180000000": 11, - "210000000": 11, - "240000000": 11, - "270000000": 11, - "300000000": 11, - "+Inf": 19 - }, - "sum": 5571080914163.27 + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false + }, + "kubernetes": { + "proxy": { + "sync": { + "rules": { + "duration": { + "us": { + "sum": 763620.9329999998, + "count": 18, + "bucket": { + "1000": 0, + "2000": 0, + "4000": 0, + "8000": 0, + "16000": 0, + "32000": 10, + "64000": 16, + "128000": 17, + "256000": 18, + "512000": 18, + "1024000": 18, + "2048000": 18, + "4096000": 18, + "8192000": 18, + "16384000": 18, + "+Inf": 18 } } } }, - "process": { - "cpu": { - "sec": 8 - }, - "memory": { - "resident": { - "bytes": 37609472 - }, - "virtual": { - "bytes": 143990784 + "networkprogramming": { + "duration": { + "us": { + "count": 19, + "bucket": { + "0": 0, + "250000": 4, + "500000": 8, + "1000000": 11, + "2000000": 11, + "3000000": 11, + "4000000": 11, + "5000000": 11, + "6000000": 11, + "7000000": 11, + "8000000": 11, + "9000000": 11, + "10000000": 11, + "11000000": 11, + "12000000": 11, + "13000000": 11, + "14000000": 11, + "15000000": 11, + "16000000": 11, + "17000000": 11, + "18000000": 11, + "19000000": 11, + "20000000": 11, + "21000000": 11, + "22000000": 11, + "23000000": 11, + "24000000": 11, + "25000000": 11, + "26000000": 11, + "27000000": 11, + "28000000": 11, + "29000000": 11, + "30000000": 11, + "31000000": 11, + "32000000": 11, + "33000000": 11, + "34000000": 11, + "35000000": 11, + "36000000": 11, + "37000000": 11, + "38000000": 11, + "39000000": 11, + "40000000": 11, + "41000000": 11, + "42000000": 11, + "43000000": 11, + "44000000": 11, + "45000000": 11, + "46000000": 11, + "47000000": 11, + "48000000": 11, + "49000000": 11, + "50000000": 11, + "51000000": 11, + "52000000": 11, + "53000000": 11, + "54000000": 11, + "55000000": 11, + "56000000": 11, + "57000000": 11, + "58000000": 11, + "59000000": 11, + "60000000": 11, + "65000000": 11, + "70000000": 11, + "75000000": 11, + "80000000": 11, + "85000000": 11, + "90000000": 11, + "95000000": 11, + "100000000": 11, + "105000000": 11, + "110000000": 11, + "115000000": 11, + "120000000": 11, + "150000000": 11, + "180000000": 11, + "210000000": 11, + "240000000": 11, + "270000000": 11, + "300000000": 11, + "+Inf": 19 + }, + "sum": 5571080914163.27 } + } + } + }, + "process": { + "cpu": { + "sec": 8 + }, + "memory": { + "resident": { + "bytes": 37609472 }, - "started": { - "sec": 1593069580.69 - }, - "fds": { - "open": { - "count": 17 - } + "virtual": { + "bytes": 143990784 + } + }, + "started": { + "sec": 1593069580.69 + }, + "fds": { + "open": { + "count": 17 } } } - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "kubernetes", - "duration": 2031254, - "dataset": "kubernetes.proxy" - }, - "metricset": { - "name": "proxy", - "period": 10000 - }, - "service": { - "address": "localhost:10249", - "type": "kubernetes" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:35:29.639Z" - ] + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "kubernetes", + "duration": 2031254, + "dataset": "kubernetes.proxy" + }, + "metricset": { + "name": "proxy", + "period": 10000 }, - "sort": [ - 1593088529639 - ] + "service": { + "address": "localhost:10249", + "type": "kubernetes" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/scheduler/sample_event.json b/packages/kubernetes/data_stream/scheduler/sample_event.json index b1fd1d5be60..a0a2cff70d1 100644 --- a/packages/kubernetes/data_stream/scheduler/sample_event.json +++ b/packages/kubernetes/data_stream/scheduler/sample_event.json @@ -1,103 +1,89 @@ { - "_index": ".ds-metrics-kubernetes.scheduler-default-000001", - "_id": "01l663IBolOt49UrTW36", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:35:59.624Z", - "agent": { - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube", - "type": "metricbeat" - }, - "dataset": { - "name": "kubernetes.scheduler", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.scheduler" - }, - "host": { - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" - }, - "name": "minikube", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ] - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "duration": 7245648, - "dataset": "kubernetes.scheduler", - "module": "kubernetes" - }, - "metricset": { - "name": "scheduler", - "period": 10000 - }, - "service": { - "address": "localhost:10251", - "type": "kubernetes" - }, - "kubernetes": { - "scheduler": { - "name": "kube-scheduler", - "leader": { - "is_master": true - } - } - } + "@timestamp": "2020-06-25T12:35:59.624Z", + "agent": { + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:35:59.624Z" + "dataset": { + "name": "kubernetes.scheduler", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.scheduler" + }, + "host": { + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "name": "minikube", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" ] }, - "sort": [ - 1593088559624 - ] + "ecs": { + "version": "1.5.0" + }, + "event": { + "duration": 7245648, + "dataset": "kubernetes.scheduler", + "module": "kubernetes" + }, + "metricset": { + "name": "scheduler", + "period": 10000 + }, + "service": { + "address": "localhost:10251", + "type": "kubernetes" + }, + "kubernetes": { + "scheduler": { + "name": "kube-scheduler", + "leader": { + "is_master": true + } + } + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_container/sample_event.json b/packages/kubernetes/data_stream/state_container/sample_event.json index fca4ce7ed73..644844c64c8 100644 --- a/packages/kubernetes/data_stream/state_container/sample_event.json +++ b/packages/kubernetes/data_stream/state_container/sample_event.json @@ -1,104 +1,90 @@ { - "_index": ".ds-metrics-kubernetes.state_container-default-000001", - "_id": "P1l663IBolOt49Ur1YbF", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:36:34.469Z", - "host": { - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false - }, - "event": { - "dataset": "kubernetes.container", - "module": "kubernetes", - "duration": 8554499 - }, - "kubernetes": { - "node": { - "name": "minikube" - }, - "labels": { - "component": "kube-scheduler", - "tier": "control-plane" - }, - "container": { - "image": "k8s.gcr.io/kube-scheduler:v1.17.0", - "name": "kube-scheduler", - "cpu": { - "request": { - "cores": 0.1 - } - }, - "status": { - "phase": "running", - "ready": true, - "restarts": 10 - }, - "id": "docker://b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" - }, - "pod": { - "name": "kube-scheduler-minikube", - "uid": "9cdbd5ea-7638-4e86-a706-a5b222d86f26" - }, - "namespace": "kube-system" - }, - "dataset": { - "name": "kubernetes.state_container", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_container", - "namespace": "default" + "@timestamp": "2020-06-25T12:36:34.469Z", + "host": { + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" }, - "ecs": { - "version": "1.5.0" + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false + }, + "event": { + "dataset": "kubernetes.container", + "module": "kubernetes", + "duration": 8554499 + }, + "kubernetes": { + "node": { + "name": "minikube" }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" + "labels": { + "component": "kube-scheduler", + "tier": "control-plane" }, "container": { - "runtime": "docker", - "id": "b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" + "image": "k8s.gcr.io/kube-scheduler:v1.17.0", + "name": "kube-scheduler", + "cpu": { + "request": { + "cores": 0.1 + } + }, + "status": { + "phase": "running", + "ready": true, + "restarts": 10 + }, + "id": "docker://b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" + "pod": { + "name": "kube-scheduler-minikube", + "uid": "9cdbd5ea-7638-4e86-a706-a5b222d86f26" }, - "metricset": { - "name": "state_container", - "period": 10000 - } + "namespace": "kube-system" + }, + "dataset": { + "name": "kubernetes.state_container", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_container", + "namespace": "default" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + }, + "container": { + "runtime": "docker", + "id": "b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:36:34.469Z" - ] + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" }, - "sort": [ - 1593088594469 - ] + "metricset": { + "name": "state_container", + "period": 10000 + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_cronjob/sample_event.json b/packages/kubernetes/data_stream/state_cronjob/sample_event.json index d06d20414dc..3ed3de3c134 100644 --- a/packages/kubernetes/data_stream/state_cronjob/sample_event.json +++ b/packages/kubernetes/data_stream/state_cronjob/sample_event.json @@ -1,90 +1,76 @@ { - "_index": ".ds-metrics-kubernetes.state_cronjob-default-000001", - "_id": "qFqA63IBolOt49Urybs0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:43:04.384Z", - "metricset": { - "name": "state_cronjob", - "period": 10000 - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" - } - }, - "event": { - "dataset": "kubernetes.cronjob", - "module": "kubernetes", - "duration": 9482053 - }, - "kubernetes": { - "namespace": "default", - "cronjob": { - "active": { - "count": 0 - }, - "is_suspended": false, - "name": "hello", - "next_schedule": { - "sec": 1593088980 - }, - "last_schedule": { - "sec": 1593088920 - }, - "created": { - "sec": 1593088862 - } + "@timestamp": "2020-06-25T12:43:04.384Z", + "metricset": { + "name": "state_cronjob", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + }, + "event": { + "dataset": "kubernetes.cronjob", + "module": "kubernetes", + "duration": 9482053 + }, + "kubernetes": { + "namespace": "default", + "cronjob": { + "active": { + "count": 0 + }, + "is_suspended": false, + "name": "hello", + "next_schedule": { + "sec": 1593088980 + }, + "last_schedule": { + "sec": 1593088920 + }, + "created": { + "sec": 1593088862 } - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_cronjob", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_cronjob" - }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:43:04.384Z" - ] + "dataset": { + "type": "metrics", + "name": "kubernetes.state_cronjob", + "namespace": "default" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.state_cronjob" }, - "sort": [ - 1593088984384 - ] + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_daemonset/sample_event.json b/packages/kubernetes/data_stream/state_daemonset/sample_event.json index a10bd1b6f74..248cc32142a 100644 --- a/packages/kubernetes/data_stream/state_daemonset/sample_event.json +++ b/packages/kubernetes/data_stream/state_daemonset/sample_event.json @@ -1,86 +1,72 @@ { - "_index": ".ds-metrics-kubernetes.state_daemonset-default-000001", - "_id": "H1l763IBolOt49UrSp72", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:37:04.455Z", - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "event": { - "module": "kubernetes", - "duration": 8648138, - "dataset": "kubernetes.daemonset" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "name": "state_daemonset", - "period": 10000 - }, - "kubernetes": { - "daemonset": { - "name": "metricbeat", - "replicas": { - "available": 1, - "desired": 1, - "ready": 1, - "unavailable": 0 - } - }, - "labels": { - "k8s-app": "metricbeat" - }, - "namespace": "kube-system" - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_daemonset", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_daemonset", - "namespace": "default" + "@timestamp": "2020-06-25T12:37:04.455Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "module": "kubernetes", + "duration": 8648138, + "dataset": "kubernetes.daemonset" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "state_daemonset", + "period": 10000 + }, + "kubernetes": { + "daemonset": { + "name": "metricbeat", + "replicas": { + "available": 1, + "desired": 1, + "ready": 1, + "unavailable": 0 + } }, - "host": { - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat" - }, - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ] + "labels": { + "k8s-app": "metricbeat" }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - } + "namespace": "kube-system" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:37:04.455Z" + "dataset": { + "type": "metrics", + "name": "kubernetes.state_daemonset", + "namespace": "default" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_daemonset", + "namespace": "default" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat" + }, + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ] }, - "sort": [ - 1593088624455 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_deployment/sample_event.json b/packages/kubernetes/data_stream/state_deployment/sample_event.json index 2ac988446db..a7d288bf9da 100644 --- a/packages/kubernetes/data_stream/state_deployment/sample_event.json +++ b/packages/kubernetes/data_stream/state_deployment/sample_event.json @@ -1,87 +1,73 @@ { - "_index": ".ds-metrics-kubernetes.state_deployment-default-000001", - "_id": "H1l763IBolOt49UrSp72", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:37:04.455Z", - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "event": { - "module": "kubernetes", - "duration": 8648138, - "dataset": "kubernetes.deployment" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "name": "state_deployment", - "period": 10000 - }, - "kubernetes": { - "deployment": { - "name": "metricbeat", - "replicas": { - "unavailable": 0, - "desired": 1, - "updated": 1, - "available": 1 - }, - "paused": false - }, - "labels": { - "k8s-app": "metricbeat" + "@timestamp": "2020-06-25T12:37:04.455Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "module": "kubernetes", + "duration": 8648138, + "dataset": "kubernetes.deployment" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "state_deployment", + "period": 10000 + }, + "kubernetes": { + "deployment": { + "name": "metricbeat", + "replicas": { + "unavailable": 0, + "desired": 1, + "updated": 1, + "available": 1 }, - "namespace": "kube-system" - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_deployment", - "namespace": "default" + "paused": false }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_deployment", - "namespace": "default" - }, - "host": { - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat" - }, - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ] + "labels": { + "k8s-app": "metricbeat" }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - } + "namespace": "kube-system" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:37:04.455Z" + "dataset": { + "type": "metrics", + "name": "kubernetes.state_deployment", + "namespace": "default" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_deployment", + "namespace": "default" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat" + }, + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ] }, - "sort": [ - 1593088624455 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_node/sample_event.json b/packages/kubernetes/data_stream/state_node/sample_event.json index 053d0e0c853..7121a3ed4fa 100644 --- a/packages/kubernetes/data_stream/state_node/sample_event.json +++ b/packages/kubernetes/data_stream/state_node/sample_event.json @@ -1,112 +1,98 @@ { - "_index": ".ds-metrics-kubernetes.state_node-default-000001", - "_id": "c1l763IBolOt49Ur58c8", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:37:44.457Z", - "ecs": { - "version": "1.5.0" - }, - "host": { - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ] - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_node" - }, - "metricset": { - "name": "state_node", - "period": 10000 + "@timestamp": "2020-06-25T12:37:44.457Z", + "ecs": { + "version": "1.5.0" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" }, - "kubernetes": { - "node": { - "pod": { - "capacity": { - "total": 110 - }, - "allocatable": { - "total": 110 - } + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ] + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.state_node" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.state_node" + }, + "metricset": { + "name": "state_node", + "period": 10000 + }, + "kubernetes": { + "node": { + "pod": { + "capacity": { + "total": 110 }, - "memory": { - "capacity": { - "bytes": 16815325184 - }, - "allocatable": { - "bytes": 16815325184 - } + "allocatable": { + "total": 110 + } + }, + "memory": { + "capacity": { + "bytes": 16815325184 }, - "cpu": { - "allocatable": { - "cores": 4 - }, - "capacity": { - "cores": 4 - } + "allocatable": { + "bytes": 16815325184 + } + }, + "cpu": { + "allocatable": { + "cores": 4 }, - "name": "minikube", - "status": { - "ready": "true", - "unschedulable": false + "capacity": { + "cores": 4 } }, - "labels": { - "kubernetes_io/arch": "amd64", - "kubernetes_io/hostname": "minikube", - "kubernetes_io/os": "linux", - "node-role_kubernetes_io/master": "", - "beta_kubernetes_io/arch": "amd64", - "beta_kubernetes_io/os": "linux" + "name": "minikube", + "status": { + "ready": "true", + "unschedulable": false } }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" - }, - "service": { - "type": "kubernetes", - "address": "kube-state-metrics:8080" - }, - "event": { - "dataset": "kubernetes.node", - "module": "kubernetes", - "duration": 8194220 + "labels": { + "kubernetes_io/arch": "amd64", + "kubernetes_io/hostname": "minikube", + "kubernetes_io/os": "linux", + "node-role_kubernetes_io/master": "", + "beta_kubernetes_io/arch": "amd64", + "beta_kubernetes_io/os": "linux" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:37:44.457Z" - ] + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "kubernetes", + "address": "kube-state-metrics:8080" }, - "sort": [ - 1593088664457 - ] + "event": { + "dataset": "kubernetes.node", + "module": "kubernetes", + "duration": 8194220 + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json b/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json index 4145ecf8d41..cfcb154d9eb 100644 --- a/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json +++ b/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json @@ -1,84 +1,70 @@ { - "_index": ".ds-metrics-kubernetes.state_persistentvolume-default-000001", - "_id": "8lqB63IBolOt49UrjOyD", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:43:54.412Z", - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "kubernetes", - "duration": 12149615, - "dataset": "kubernetes.persistentvolume" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - }, - "kubernetes": { - "persistentvolume": { - "capacity": { - "bytes": 10737418240 - }, - "phase": "Bound", - "storage_class": "manual", - "name": "task-pv-volume" - }, - "labels": { - "type": "local" - } - }, - "dataset": { - "name": "kubernetes.state_persistentvolume", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolume", - "namespace": "default" - }, - "host": { - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" + "@timestamp": "2020-06-25T12:43:54.412Z", + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "kubernetes", + "duration": 12149615, + "dataset": "kubernetes.persistentvolume" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + }, + "kubernetes": { + "persistentvolume": { + "capacity": { + "bytes": 10737418240 }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "containerized": false + "phase": "Bound", + "storage_class": "manual", + "name": "task-pv-volume" }, - "metricset": { - "period": 10000, - "name": "state_persistentvolume" - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" + "labels": { + "type": "local" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:43:54.412Z" - ] + "dataset": { + "name": "kubernetes.state_persistentvolume", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_persistentvolume", + "namespace": "default" + }, + "host": { + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false + }, + "metricset": { + "period": 10000, + "name": "state_persistentvolume" }, - "sort": [ - 1593089034412 - ] + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json b/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json index 9bed776641d..6d63fa4e49f 100644 --- a/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json +++ b/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json @@ -1,84 +1,70 @@ { - "_index": ".ds-metrics-kubernetes.state_persistentvolumeclaim-default-000001", - "_id": "6FuC63IBolOt49UrTxrR", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:44:44.418Z", - "event": { - "dataset": "kubernetes.persistentvolumeclaim", - "module": "kubernetes", - "duration": 5698588 - }, - "metricset": { - "name": "state_persistentvolumeclaim", - "period": 10000 - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "kubernetes": { - "namespace": "default", - "persistentvolumeclaim": { - "phase": "Bound", - "storage_class": "manual", - "volume_name": "task-pv-volume", - "name": "task-pv-claim", - "request_storage": { - "bytes": 3221225472 - }, - "access_mode": "ReadWriteOnce" - } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_persistentvolumeclaim" - }, - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolumeclaim", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" + "@timestamp": "2020-06-25T12:44:44.418Z", + "event": { + "dataset": "kubernetes.persistentvolumeclaim", + "module": "kubernetes", + "duration": 5698588 + }, + "metricset": { + "name": "state_persistentvolumeclaim", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "kubernetes": { + "namespace": "default", + "persistentvolumeclaim": { + "phase": "Bound", + "storage_class": "manual", + "volume_name": "task-pv-volume", + "name": "task-pv-claim", + "request_storage": { + "bytes": 3221225472 }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64" + "access_mode": "ReadWriteOnce" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:44:44.418Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.state_persistentvolumeclaim" + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" }, - "sort": [ - 1593089084418 - ] + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_persistentvolumeclaim", + "namespace": "default" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_pod/sample_event.json b/packages/kubernetes/data_stream/state_pod/sample_event.json index 9036e5c8a13..82194c27b65 100644 --- a/packages/kubernetes/data_stream/state_pod/sample_event.json +++ b/packages/kubernetes/data_stream/state_pod/sample_event.json @@ -1,93 +1,79 @@ { - "_index": ".ds-metrics-kubernetes.state_pod-default-000001", - "_id": "YVl863IBolOt49UrqueH", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:38:34.469Z", - "dataset": { - "name": "kubernetes.state_pod", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_pod" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" - } - }, - "event": { - "duration": 10777415, - "dataset": "kubernetes.pod", - "module": "kubernetes" - }, - "service": { - "type": "kubernetes", - "address": "kube-state-metrics:8080" - }, - "kubernetes": { - "pod": { - "name": "filebeat-dqzzz", - "status": { - "ready": "true", - "scheduled": "true", - "phase": "running" - }, - "host_ip": "192.168.64.10", - "ip": "192.168.64.10", - "uid": "a5f1d3c9-40b6-4182-823b-dd5ff9832279" - }, - "namespace": "kube-system", - "node": { - "name": "minikube" + "@timestamp": "2020-06-25T12:38:34.469Z", + "dataset": { + "name": "kubernetes.state_pod", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.state_pod" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + } + }, + "event": { + "duration": 10777415, + "dataset": "kubernetes.pod", + "module": "kubernetes" + }, + "service": { + "type": "kubernetes", + "address": "kube-state-metrics:8080" + }, + "kubernetes": { + "pod": { + "name": "filebeat-dqzzz", + "status": { + "ready": "true", + "scheduled": "true", + "phase": "running" }, - "labels": { - "controller-revision-hash": "85649b9ddb", - "k8s-app": "filebeat", - "pod-template-generation": "1" - } + "host_ip": "192.168.64.10", + "ip": "192.168.64.10", + "uid": "a5f1d3c9-40b6-4182-823b-dd5ff9832279" }, - "agent": { - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" + "namespace": "kube-system", + "node": { + "name": "minikube" }, - "metricset": { - "period": 10000, - "name": "state_pod" + "labels": { + "controller-revision-hash": "85649b9ddb", + "k8s-app": "filebeat", + "pod-template-generation": "1" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:38:34.469Z" - ] + "agent": { + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" }, - "sort": [ - 1593088714469 - ] + "metricset": { + "period": 10000, + "name": "state_pod" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_replicaset/sample_event.json b/packages/kubernetes/data_stream/state_replicaset/sample_event.json index e40938a5d00..6cc5548f5a7 100644 --- a/packages/kubernetes/data_stream/state_replicaset/sample_event.json +++ b/packages/kubernetes/data_stream/state_replicaset/sample_event.json @@ -1,93 +1,79 @@ { - "_index": ".ds-metrics-kubernetes.state_replicaset-default-000001", - "_id": "U1l863IBolOt49Ur-Pu2", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:38:54.482Z", - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "metricset": { - "period": 10000, - "name": "state_replicaset" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_replicaset" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_replicaset", - "namespace": "default" - }, - "event": { - "module": "kubernetes", - "duration": 5456128, - "dataset": "kubernetes.replicaset" - }, - "kubernetes": { - "namespace": "kube-system", - "replicaset": { - "name": "nginx-ingress-controller-6fc5bcc8c9", - "replicas": { - "labeled": 1, - "ready": 1, - "available": 1, - "observed": 1, - "desired": 1 - } - }, - "deployment": { - "name": "nginx-ingress-controller" - }, - "labels": { - "app_kubernetes_io/part-of": "kube-system", - "pod-template-hash": "6fc5bcc8c9", - "addonmanager_kubernetes_io/mode": "Reconcile", - "app_kubernetes_io/name": "nginx-ingress-controller" + "@timestamp": "2020-06-25T12:38:54.482Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "metricset": { + "period": 10000, + "name": "state_replicaset" + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.state_replicaset" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_replicaset", + "namespace": "default" + }, + "event": { + "module": "kubernetes", + "duration": 5456128, + "dataset": "kubernetes.replicaset" + }, + "kubernetes": { + "namespace": "kube-system", + "replicaset": { + "name": "nginx-ingress-controller-6fc5bcc8c9", + "replicas": { + "labeled": 1, + "ready": 1, + "available": 1, + "observed": 1, + "desired": 1 } }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" + "deployment": { + "name": "nginx-ingress-controller" }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - }, - "id": "b0e83d397c054b8a99a431072fe4617b" + "labels": { + "app_kubernetes_io/part-of": "kube-system", + "pod-template-hash": "6fc5bcc8c9", + "addonmanager_kubernetes_io/mode": "Reconcile", + "app_kubernetes_io/name": "nginx-ingress-controller" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:38:54.482Z" - ] + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1593088734482 - ] + "host": { + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_resourcequota/sample_event.json b/packages/kubernetes/data_stream/state_resourcequota/sample_event.json index de22293ec37..73a1813a8fd 100644 --- a/packages/kubernetes/data_stream/state_resourcequota/sample_event.json +++ b/packages/kubernetes/data_stream/state_resourcequota/sample_event.json @@ -1,80 +1,66 @@ { - "_index": ".ds-metrics-kubernetes.state_resourcequota-default-000001", - "_id": "4FuC63IBolOt49UrnSHz", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:45:04.416Z", - "metricset": { - "name": "state_resourcequota", - "period": 10000 - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_resourcequota", - "namespace": "default" - }, - "host": { - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ] - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "event": { - "dataset": "kubernetes.resourcequota", - "module": "kubernetes", - "duration": 6324269 - }, - "agent": { - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" - }, - "ecs": { - "version": "1.5.0" - }, - "kubernetes": { - "namespace": "quota-object-example", - "resourcequota": { - "name": "object-quota-demo", - "resource": "persistentvolumeclaims", - "type": "hard", - "quota": 1 - } - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_resourcequota", - "namespace": "default" - } + "@timestamp": "2020-06-25T12:45:04.416Z", + "metricset": { + "name": "state_resourcequota", + "period": 10000 }, - "fields": { - "@timestamp": [ - "2020-06-25T12:45:04.416Z" + "dataset": { + "type": "metrics", + "name": "kubernetes.state_resourcequota", + "namespace": "default" + }, + "host": { + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" ] }, - "sort": [ - 1593089104416 - ] + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.resourcequota", + "module": "kubernetes", + "duration": 6324269 + }, + "agent": { + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" + }, + "ecs": { + "version": "1.5.0" + }, + "kubernetes": { + "namespace": "quota-object-example", + "resourcequota": { + "name": "object-quota-demo", + "resource": "persistentvolumeclaims", + "type": "hard", + "quota": 1 + } + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_resourcequota", + "namespace": "default" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_service/sample_event.json b/packages/kubernetes/data_stream/state_service/sample_event.json index 5ffa06ad90e..13fcd4e303e 100644 --- a/packages/kubernetes/data_stream/state_service/sample_event.json +++ b/packages/kubernetes/data_stream/state_service/sample_event.json @@ -1,89 +1,72 @@ { - "_index": ".ds-metrics-kubernetes.state_service-default-000001", - "_id": "Elp963IBolOt49UrbRPd", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:24.389Z", - "kubernetes": { - "labels": { - "kubernetes_io_minikube_addons_endpoint": "metrics-server", - "kubernetes_io_name": "Metrics-server", - "addonmanager_kubernetes_io_mode": "Reconcile", - "kubernetes_io_minikube_addons": "metrics-server" - }, - "service": { - "name": "metrics-server", - "created": "2020-06-10T09:02:27.000Z", - "cluster_ip": "10.96.124.248", - "type": "ClusterIP" - }, - "namespace": "kube-system" - }, - "event": { - "dataset": "kubernetes.service", - "module": "kubernetes", - "duration": 10966648 - }, - "metricset": { - "name": "state_service", - "period": 10000 - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - } - }, - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-06-25T12:39:24.389Z", + "kubernetes": { + "labels": { + "kubernetes_io_minikube_addons_endpoint": "metrics-server", + "kubernetes_io_name": "Metrics-server", + "addonmanager_kubernetes_io_mode": "Reconcile", + "kubernetes_io_minikube_addons": "metrics-server" }, "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" + "name": "metrics-server", + "created": "2020-06-10T09:02:27.000Z", + "cluster_ip": "10.96.124.248", + "type": "ClusterIP" }, - "stream": { - "dataset": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" - } + "namespace": "kube-system" + }, + "event": { + "dataset": "kubernetes.service", + "module": "kubernetes", + "duration": 10966648 }, - "fields": { - "@timestamp": [ - "2020-06-25T12:39:24.389Z" + "metricset": { + "name": "state_service", + "period": 10000 + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" ], - "kubernetes.service.created": [ - "2020-06-10T09:02:27.000Z" - ] + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "dataset": { + "name": "kubernetes.state_service", + "namespace": "default", + "type": "metrics" }, - "sort": [ - 1593088764389 - ] + "stream": { + "dataset": "kubernetes.state_service", + "namespace": "default", + "type": "metrics" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_statefulset/sample_event.json b/packages/kubernetes/data_stream/state_statefulset/sample_event.json index 02506e87d3e..16afea87e60 100644 --- a/packages/kubernetes/data_stream/state_statefulset/sample_event.json +++ b/packages/kubernetes/data_stream/state_statefulset/sample_event.json @@ -1,89 +1,72 @@ { - "_index": ".ds-metrics-kubernetes.state_statefulset-default-000001", - "_id": "Elp963IBolOt49UrbRPd", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:24.389Z", - "kubernetes": { - "namespace": "default", - "statefulset": { - "created": 1511989697, - "generation": { - "desired": 4, - "observed": 2 - }, - "name": "mysql", - "replicas": { - "desired": 5, - "observed": 2 - } + "@timestamp": "2020-06-25T12:39:24.389Z", + "kubernetes": { + "namespace": "default", + "statefulset": { + "created": 1511989697, + "generation": { + "desired": 4, + "observed": 2 + }, + "name": "mysql", + "replicas": { + "desired": 5, + "observed": 2 } - }, - "event": { - "dataset": "kubernetes.statefulset", - "module": "kubernetes", - "duration": 10966648 - }, - "metricset": { - "name": "state_statefulset", - "period": 10000 - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - } - }, - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "ecs": { - "version": "1.5.0" - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:39:24.389Z" + "event": { + "dataset": "kubernetes.statefulset", + "module": "kubernetes", + "duration": 10966648 + }, + "metricset": { + "name": "state_statefulset", + "period": 10000 + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.statefulset.created": [ - "2020-06-10T09:02:27.000Z" - ] + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "dataset": { + "name": "kubernetes.state_statefulset", + "namespace": "default", + "type": "metrics" }, - "sort": [ - 1593088764389 - ] + "stream": { + "dataset": "kubernetes.state_statefulset", + "namespace": "default", + "type": "metrics" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_storageclass/sample_event.json b/packages/kubernetes/data_stream/state_storageclass/sample_event.json index dfb527e14de..1f5d878752b 100644 --- a/packages/kubernetes/data_stream/state_storageclass/sample_event.json +++ b/packages/kubernetes/data_stream/state_storageclass/sample_event.json @@ -1,86 +1,69 @@ { - "_index": ".ds-metrics-kubernetes.state_storageclass-default-000001", - "_id": "KFp963IBolOt49UruyX3", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:44.399Z", - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "kubernetes": { - "storageclass": { - "provisioner": "k8s.io/minikube-hostpath", - "reclaim_policy": "Delete", - "volume_binding_mode": "Immediate", - "name": "standard", - "created": "2020-06-10T09:02:27.000Z" - }, - "labels": { - "addonmanager_kubernetes_io_mode": "EnsureExists" - } - }, - "dataset": { - "name": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, - "host": { - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ] - }, - "event": { - "module": "kubernetes", - "duration": 5713503, - "dataset": "kubernetes.storageclass" - }, - "metricset": { - "name": "state_storageclass", - "period": 10000 - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" + "@timestamp": "2020-06-25T12:39:44.399Z", + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "kubernetes": { + "storageclass": { + "provisioner": "k8s.io/minikube-hostpath", + "reclaim_policy": "Delete", + "volume_binding_mode": "Immediate", + "name": "standard", + "created": "2020-06-10T09:02:27.000Z" }, - "ecs": { - "version": "1.5.0" + "labels": { + "addonmanager_kubernetes_io_mode": "EnsureExists" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:39:44.399Z" + "dataset": { + "name": "kubernetes.state_storageclass", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "dataset": "kubernetes.state_storageclass", + "namespace": "default", + "type": "metrics" + }, + "host": { + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.storageclass.created": [ - "2020-06-10T09:02:27.000Z" + "mac": [ + "02:42:ac:11:00:0b" ] }, - "sort": [ - 1593088784399 - ] + "event": { + "module": "kubernetes", + "duration": 5713503, + "dataset": "kubernetes.storageclass" + }, + "metricset": { + "name": "state_storageclass", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/system/sample_event.json b/packages/kubernetes/data_stream/system/sample_event.json index 93607c5c09f..59ed7ff327a 100644 --- a/packages/kubernetes/data_stream/system/sample_event.json +++ b/packages/kubernetes/data_stream/system/sample_event.json @@ -1,128 +1,111 @@ { - "_index": ".ds-metrics-kubernetes.system-default-000001", - "_id": "sVp963IBolOt49Ur9yyT", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:59.647Z", - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.system" - }, - "service": { - "address": "minikube:10250", - "type": "kubernetes" - }, - "event": { - "duration": 20012905, - "dataset": "kubernetes.system", - "module": "kubernetes" - }, - "stream": { - "dataset": "kubernetes.system", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "name": "minikube", - "architecture": "x86_64", - "os": { - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ] + "@timestamp": "2020-06-25T12:39:59.647Z", + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.system" + }, + "service": { + "address": "minikube:10250", + "type": "kubernetes" + }, + "event": { + "duration": 20012905, + "dataset": "kubernetes.system", + "module": "kubernetes" + }, + "stream": { + "dataset": "kubernetes.system", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "name": "minikube", + "architecture": "x86_64", + "os": { + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos" }, - "agent": { - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube", - "type": "metricbeat", - "version": "8.0.0" + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ] + }, + "agent": { + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0" + }, + "kubernetes": { + "node": { + "name": "minikube" }, - "kubernetes": { - "node": { - "name": "minikube" - }, - "system": { - "container": "runtime", - "cpu": { - "usage": { - "nanocores": 35779815, - "core": { - "ns": 530899961233 - } + "system": { + "container": "runtime", + "cpu": { + "usage": { + "nanocores": 35779815, + "core": { + "ns": 530899961233 } + } + }, + "memory": { + "pagefaults": 12944019, + "majorpagefaults": 99, + "usage": { + "bytes": 198279168 }, - "memory": { - "pagefaults": 12944019, - "majorpagefaults": 99, - "usage": { - "bytes": 198279168 - }, - "workingset": { - "bytes": 178794496 - }, - "rss": { - "bytes": 125259776 - } + "workingset": { + "bytes": 178794496 }, - "start_time": "2020-06-25T07:19:32Z" - } - }, - "metricset": { - "name": "system", - "period": 10000 + "rss": { + "bytes": 125259776 + } + }, + "start_time": "2020-06-25T07:19:32Z" } }, - "fields": { - "kubernetes.system.start_time": [ - "2020-06-25T07:19:32.000Z" - ], - "@timestamp": [ - "2020-06-25T12:39:59.647Z" - ] - }, - "sort": [ - 1593088799647 - ] + "metricset": { + "name": "system", + "period": 10000 + } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/volume/sample_event.json b/packages/kubernetes/data_stream/volume/sample_event.json index c36fc2f6ab2..859a01a72ee 100644 --- a/packages/kubernetes/data_stream/volume/sample_event.json +++ b/packages/kubernetes/data_stream/volume/sample_event.json @@ -1,123 +1,109 @@ { - "_index": ".ds-metrics-kubernetes.volume-default-000001", - "_id": "b1p-63IBolOt49UrRT-d", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:40:19.649Z", - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "name": "volume", - "period": 10000 - }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" + "@timestamp": "2020-06-25T12:40:19.649Z", + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "volume", + "period": 10000 + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "kubernetes": { + "pod": { + "name": "metricbeat-g9fc6" }, - "kubernetes": { - "pod": { - "name": "metricbeat-g9fc6" - }, - "volume": { - "name": "config", - "fs": { - "inodes": { - "used": 5, - "free": 9549949, - "count": 9768928 - }, - "available": { - "bytes": 7719858176 - }, - "capacity": { - "bytes": 17361141760 - }, - "used": { - "bytes": 12288 - } + "volume": { + "name": "config", + "fs": { + "inodes": { + "used": 5, + "free": 9549949, + "count": 9768928 + }, + "available": { + "bytes": 7719858176 + }, + "capacity": { + "bytes": 17361141760 + }, + "used": { + "bytes": 12288 } - }, - "namespace": "kube-system", - "node": { - "name": "minikube" } }, - "dataset": { - "type": "metrics", - "name": "kubernetes.volume", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.volume" - }, - "host": { - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "name": "minikube", - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "namespace": "kube-system", + "node": { "name": "minikube" - }, - "event": { - "dataset": "kubernetes.volume", - "module": "kubernetes", - "duration": 12481688 } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:40:19.649Z" - ] + "dataset": { + "type": "metrics", + "name": "kubernetes.volume", + "namespace": "default" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.volume" + }, + "host": { + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "name": "minikube", + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" }, - "sort": [ - 1593088819649 - ] + "event": { + "dataset": "kubernetes.volume", + "module": "kubernetes", + "duration": 12481688 + } } \ No newline at end of file diff --git a/packages/kubernetes/docs/README.md b/packages/kubernetes/docs/README.md index 12da786248c..1483d52e86f 100644 --- a/packages/kubernetes/docs/README.md +++ b/packages/kubernetes/docs/README.md @@ -114,90 +114,76 @@ An example event for `apiserver` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.apiserver-default-000001", - "_id": "XVh163IBolOt49UrV2yq", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:30:34.616Z", - "metricset": { - "name": "apiserver", - "period": 30000 - }, - "service": { - "address": "10.96.0.1:443", - "type": "kubernetes" - }, - "event": { - "dataset": "kubernetes.apiserver", - "module": "kubernetes", - "duration": 114780772 - }, - "kubernetes": { - "apiserver": { - "request": { - "client": "metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format", - "version": "v1", - "count": 3, - "scope": "cluster", - "content_type": "application/vnd.kubernetes.protobuf", - "code": "200", - "verb": "LIST", - "component": "apiserver", - "resource": "nodes" - } - } - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.apiserver" - }, - "stream": { - "dataset": "kubernetes.apiserver", - "namespace": "default", - "type": "metrics" - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" + "@timestamp": "2020-06-25T12:30:34.616Z", + "metricset": { + "name": "apiserver", + "period": 30000 + }, + "service": { + "address": "10.96.0.1:443", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.apiserver", + "module": "kubernetes", + "duration": 114780772 + }, + "kubernetes": { + "apiserver": { + "request": { + "client": "metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format", + "version": "v1", + "count": 3, + "scope": "cluster", + "content_type": "application/vnd.kubernetes.protobuf", + "code": "200", + "verb": "LIST", + "component": "apiserver", + "resource": "nodes" } } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:30:34.616Z" - ] + "ecs": { + "version": "1.5.0" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" }, - "sort": [ - 1593088234616 - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.apiserver" + }, + "stream": { + "dataset": "kubernetes.apiserver", + "namespace": "default", + "type": "metrics" + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + } } ``` @@ -295,181 +281,164 @@ An example event for `container` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.container-default-000001", - "_id": "y1h363IBolOt49UrGcjO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:32:29.748Z", - "kubernetes": { - "namespace": "kube-system", - "node": { - "name": "minikube" + "@timestamp": "2020-06-25T12:32:29.748Z", + "kubernetes": { + "namespace": "kube-system", + "node": { + "name": "minikube" + }, + "pod": { + "name": "metricbeat-g9fc6" + }, + "container": { + "rootfs": { + "used": { + "bytes": 61440 + }, + "inodes": { + "used": 17 + }, + "available": { + "bytes": 6724222976 + }, + "capacity": { + "bytes": 17361141760 + } }, - "pod": { - "name": "metricbeat-g9fc6" + "logs": { + "used": { + "bytes": 1617920 + }, + "inodes": { + "count": 9768928, + "used": 223910, + "free": 9545018 + }, + "available": { + "bytes": 6724222976 + }, + "capacity": { + "bytes": 17361141760 + } }, - "container": { - "rootfs": { - "used": { - "bytes": 61440 - }, - "inodes": { - "used": 17 + "start_time": "2020-06-25T07:19:37Z", + "name": "metricbeat", + "cpu": { + "usage": { + "node": { + "pct": 0.00015289625 }, - "available": { - "bytes": 6724222976 + "limit": { + "pct": 0.00015289625 }, - "capacity": { - "bytes": 17361141760 + "nanocores": 611585, + "core": { + "ns": 12206519774 } + } + }, + "memory": { + "pagefaults": 10164, + "majorpagefaults": 528, + "available": { + "bytes": 188600320 }, - "logs": { - "used": { - "bytes": 1617920 - }, - "inodes": { - "count": 9768928, - "used": 223910, - "free": 9545018 + "usage": { + "limit": { + "pct": 0.005608354460473573 }, - "available": { - "bytes": 6724222976 - }, - "capacity": { - "bytes": 17361141760 + "bytes": 94306304, + "node": { + "pct": 0.005608354460473573 } }, - "start_time": "2020-06-25T07:19:37Z", - "name": "metricbeat", - "cpu": { - "usage": { - "node": { - "pct": 0.00015289625 - }, - "limit": { - "pct": 0.00015289625 - }, - "nanocores": 611585, - "core": { - "ns": 12206519774 - } - } + "workingset": { + "bytes": 21114880 }, - "memory": { - "pagefaults": 10164, - "majorpagefaults": 528, - "available": { - "bytes": 188600320 - }, - "usage": { - "limit": { - "pct": 0.005608354460473573 - }, - "bytes": 94306304, - "node": { - "pct": 0.005608354460473573 - } - }, - "workingset": { - "bytes": 21114880 - }, - "rss": { - "bytes": 18386944 - } + "rss": { + "bytes": 18386944 } } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.container" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.container" - }, - "host": { - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" - }, - "name": "minikube", - "id": "b0e83d397c054b8a99a431072fe4617b" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube" - }, - "metricset": { - "period": 10000, - "name": "container" - }, - "service": { - "address": "minikube:10250", - "type": "kubernetes" - }, - "event": { - "dataset": "kubernetes.container", - "module": "kubernetes", - "duration": 11091346 - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:32:29.748Z" + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.container" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.container" + }, + "host": { + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" ], - "kubernetes.container.start_time": [ - "2020-06-25T07:19:37.000Z" - ] + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + }, + "name": "minikube", + "id": "b0e83d397c054b8a99a431072fe4617b" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + }, + "metricset": { + "period": 10000, + "name": "container" }, - "sort": [ - 1593088349748 - ] + "service": { + "address": "minikube:10250", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.container", + "module": "kubernetes", + "duration": 11091346 + }, + "ecs": { + "version": "1.5.0" + } } ``` @@ -559,121 +528,107 @@ An example event for `controllermanager` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.controllermanager-default-000001", - "_id": "qFh463IBolOt49UrBPYP", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:33:29.643Z", - "kubernetes": { - "controllermanager": { - "workqueue": { - "unfinished": { - "sec": 0 - }, - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - } + "@timestamp": "2020-06-25T12:33:29.643Z", + "kubernetes": { + "controllermanager": { + "workqueue": { + "unfinished": { + "sec": 0 }, - "name": "certificate" - } - }, - "event": { - "dataset": "kubernetes.controllermanager", - "module": "kubernetes", - "duration": 8893806 - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.controllermanager" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + } }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "name": "minikube" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube", - "type": "metricbeat" - }, - "metricset": { - "period": 10000, - "name": "controllermanager" - }, - "service": { - "address": "localhost:10252", - "type": "kubernetes" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.controllermanager" + "name": "certificate" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:33:29.643Z" - ] + "event": { + "dataset": "kubernetes.controllermanager", + "module": "kubernetes", + "duration": 8893806 + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.controllermanager" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "name": "minikube" }, - "sort": [ - 1593088409643 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat" + }, + "metricset": { + "period": 10000, + "name": "controllermanager" + }, + "service": { + "address": "localhost:10252", + "type": "kubernetes" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.controllermanager" + } } ``` @@ -760,115 +715,92 @@ An example event for `event` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.event-default-000001", - "_id": "EVh163IBolOt49UrPGji", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:30:27.575Z", - "metricset": { - "name": "event" - }, - "stream": { - "dataset": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc" - }, - "ecs": { - "version": "1.5.0" - }, + "@timestamp": "2020-06-25T12:30:27.575Z", + "metricset": { + "name": "event" + }, + "stream": { + "dataset": "kubernetes.event", + "namespace": "default", + "type": "metrics" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "kubernetes.event", + "module": "kubernetes" + }, + "service": { + "type": "kubernetes" + }, + "kubernetes": { "event": { - "dataset": "kubernetes.event", - "module": "kubernetes" - }, - "service": { - "type": "kubernetes" - }, - "kubernetes": { - "event": { - "metadata": { - "uid": "604e39e0-862f-4615-9cec-8cb62299dea3", - "resource_version": "485630", - "timestamp": { - "created": "2020-06-25T07:20:25.000Z" - }, - "name": "monitor.161bb862545e3099", - "namespace": "beats", - "self_link": "/api/v1/namespaces/beats/events/monitor.161bb862545e3099", - "generate_name": "" - }, + "metadata": { + "uid": "604e39e0-862f-4615-9cec-8cb62299dea3", + "resource_version": "485630", "timestamp": { - "first_occurrence": "2020-06-25T07:20:25.000Z", - "last_occurrence": "2020-06-25T12:30:27.000Z" - }, - "message": "Failed to find referenced backend beats/monitor: Elasticsearch.elasticsearch.k8s.elastic.co \"monitor\" not found", - "reason": "AssociationError", - "type": "Warning", - "count": 1861, - "source": { - "host": "", - "component": "kibana-association-controller" + "created": "2020-06-25T07:20:25.000Z" }, - "involved_object": { - "api_version": "kibana.k8s.elastic.co/v1", - "resource_version": "101842", - "name": "monitor", - "kind": "Kibana", - "uid": "45a19de5-5eef-4090-a2d3-dbceb0a28af8" - } - } - }, - "dataset": { - "name": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" + "name": "monitor.161bb862545e3099", + "namespace": "beats", + "self_link": "/api/v1/namespaces/beats/events/monitor.161bb862545e3099", + "generate_name": "" + }, + "timestamp": { + "first_occurrence": "2020-06-25T07:20:25.000Z", + "last_occurrence": "2020-06-25T12:30:27.000Z" + }, + "message": "Failed to find referenced backend beats/monitor: Elasticsearch.elasticsearch.k8s.elastic.co \"monitor\" not found", + "reason": "AssociationError", + "type": "Warning", + "count": 1861, + "source": { + "host": "", + "component": "kibana-association-controller" + }, + "involved_object": { + "api_version": "kibana.k8s.elastic.co/v1", + "resource_version": "101842", + "name": "monitor", + "kind": "Kibana", + "uid": "45a19de5-5eef-4090-a2d3-dbceb0a28af8" } } }, - "fields": { - "kubernetes.event.timestamp.first_occurrence": [ - "2020-06-25T07:20:25.000Z" - ], - "kubernetes.event.timestamp.last_occurrence": [ - "2020-06-25T12:30:27.000Z" + "dataset": { + "name": "kubernetes.event", + "namespace": "default", + "type": "metrics" + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.event.metadata.timestamp.created": [ - "2020-06-25T07:20:25.000Z" + "mac": [ + "02:42:ac:11:00:0b" ], - "@timestamp": [ - "2020-06-25T12:30:27.575Z" - ] - }, - "sort": [ - 1593088227575 - ] + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + } } ``` @@ -940,179 +872,162 @@ An example event for `node` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.node-default-000001", - "_id": "Gll563IBolOt49UrFS2Q", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:34:39.723Z", - "event": { - "dataset": "kubernetes.node", - "module": "kubernetes", - "duration": 13042307 - }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" - }, - "host": { - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "name": "minikube", - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" + "@timestamp": "2020-06-25T12:34:39.723Z", + "event": { + "dataset": "kubernetes.node", + "module": "kubernetes", + "duration": 13042307 + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "host": { + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b" + }, + "metricset": { + "name": "node", + "period": 10000 + }, + "kubernetes": { + "labels": { + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "amd64", + "kubernetes_io/hostname": "minikube", + "kubernetes_io/os": "linux", + "node-role_kubernetes_io/master": "", + "beta_kubernetes_io/arch": "amd64" + }, + "node": { + "memory": { + "available": { + "bytes": 12746428416 + }, + "usage": { + "bytes": 5670916096 + }, + "workingset": { + "bytes": 4068896768 + }, + "rss": { + "bytes": 3252125696 + }, + "pagefaults": 31680, + "majorpagefaults": 0 }, - "id": "b0e83d397c054b8a99a431072fe4617b" - }, - "metricset": { - "name": "node", - "period": 10000 - }, - "kubernetes": { - "labels": { - "beta_kubernetes_io/os": "linux", - "kubernetes_io/arch": "amd64", - "kubernetes_io/hostname": "minikube", - "kubernetes_io/os": "linux", - "node-role_kubernetes_io/master": "", - "beta_kubernetes_io/arch": "amd64" + "network": { + "rx": { + "bytes": 107077476, + "errors": 0 + }, + "tx": { + "bytes": 67457933, + "errors": 0 + } }, - "node": { - "memory": { - "available": { - "bytes": 12746428416 - }, - "usage": { - "bytes": 5670916096 - }, - "workingset": { - "bytes": 4068896768 - }, - "rss": { - "bytes": 3252125696 - }, - "pagefaults": 31680, - "majorpagefaults": 0 + "fs": { + "available": { + "bytes": 6655090688 }, - "network": { - "rx": { - "bytes": 107077476, - "errors": 0 - }, - "tx": { - "bytes": 67457933, - "errors": 0 - } + "capacity": { + "bytes": 17361141760 }, - "fs": { - "available": { - "bytes": 6655090688 - }, + "used": { + "bytes": 9689358336 + }, + "inodes": { + "count": 9768928, + "used": 224151, + "free": 9544777 + } + }, + "runtime": { + "imagefs": { "capacity": { "bytes": 17361141760 }, "used": { - "bytes": 9689358336 + "bytes": 8719928568 }, - "inodes": { - "count": 9768928, - "used": 224151, - "free": 9544777 - } - }, - "runtime": { - "imagefs": { - "capacity": { - "bytes": 17361141760 - }, - "used": { - "bytes": 8719928568 - }, - "available": { - "bytes": 6655090688 - } - } - }, - "start_time": "2020-06-25T07:18:38Z", - "name": "minikube", - "cpu": { - "usage": { - "core": { - "ns": 6136184971873 - }, - "nanocores": 455263291 + "available": { + "bytes": 6655090688 } } - } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.node" - }, - "agent": { + }, + "start_time": "2020-06-25T07:18:38Z", "name": "minikube", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" - }, - "ecs": { - "version": "1.5.0" + "cpu": { + "usage": { + "core": { + "ns": 6136184971873 + }, + "nanocores": 455263291 + } + } } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:34:39.723Z" - ], - "kubernetes.node.start_time": [ - "2020-06-25T07:18:38.000Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.node" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.node" + }, + "agent": { + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" }, - "sort": [ - 1593088479723 - ] + "ecs": { + "version": "1.5.0" + } } ``` @@ -1201,156 +1116,139 @@ An example event for `pod` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.pod-default-000001", - "_id": "4Vl563IBolOt49UrYz6x", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:34:59.729Z", - "kubernetes": { - "pod": { - "memory": { - "rss": { - "bytes": 7823360 - }, - "page_faults": 5742, - "major_page_faults": 0, - "usage": { - "limit": { - "pct": 0.0008033509820466402 - }, - "bytes": 13508608, - "node": { - "pct": 0.0008033509820466402 - } - }, - "available": { - "bytes": 0 - }, - "working_set": { - "bytes": 8556544 - } + "@timestamp": "2020-06-25T12:34:59.729Z", + "kubernetes": { + "pod": { + "memory": { + "rss": { + "bytes": 7823360 }, - "network": { - "rx": { - "bytes": 25671624, - "errors": 0 + "page_faults": 5742, + "major_page_faults": 0, + "usage": { + "limit": { + "pct": 0.0008033509820466402 }, - "tx": { - "errors": 0, - "bytes": 1092900259 + "bytes": 13508608, + "node": { + "pct": 0.0008033509820466402 } }, - "start_time": "2020-06-18T11:12:58Z", - "name": "kube-state-metrics-57cd6fdf9-hd959", - "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", - "cpu": { - "usage": { - "nanocores": 2811918, - "node": { - "pct": 0.0007029795 - }, - "limit": { - "pct": 0.0007029795 - } - } + "available": { + "bytes": 0 + }, + "working_set": { + "bytes": 8556544 } }, - "namespace": "kube-system", - "node": { - "name": "minikube" - } - }, - "event": { - "duration": 20735189, - "dataset": "kubernetes.pod", - "module": "kubernetes" - }, - "stream": { - "dataset": "kubernetes.pod", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "period": 10000, - "name": "pod" - }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.pod", - "namespace": "default" - }, - "host": { - "name": "minikube", - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" + "network": { + "rx": { + "bytes": 25671624, + "errors": 0 + }, + "tx": { + "errors": 0, + "bytes": 1092900259 + } }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ] + "start_time": "2020-06-18T11:12:58Z", + "name": "kube-state-metrics-57cd6fdf9-hd959", + "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", + "cpu": { + "usage": { + "nanocores": 2811918, + "node": { + "pct": 0.0007029795 + }, + "limit": { + "pct": 0.0007029795 + } + } + } }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "namespace": "kube-system", + "node": { "name": "minikube" } }, - "fields": { - "kubernetes.pod.start_time": [ - "2020-06-18T11:12:58.000Z" + "event": { + "duration": 20735189, + "dataset": "kubernetes.pod", + "module": "kubernetes" + }, + "stream": { + "dataset": "kubernetes.pod", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "period": 10000, + "name": "pod" + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "dataset": { + "type": "metrics", + "name": "kubernetes.pod", + "namespace": "default" + }, + "host": { + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" ], - "@timestamp": [ - "2020-06-25T12:34:59.729Z" + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" ] }, - "sort": [ - 1593088499729 - ] + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + } } ``` @@ -1433,244 +1331,230 @@ An example event for `proxy` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.proxy-default-000001", - "_id": "Z1l563IBolOt49Ur2FXO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:35:29.639Z", - "agent": { - "name": "minikube", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.proxy" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.proxy", - "namespace": "default" - }, - "host": { - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "name": "minikube", - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false - }, - "kubernetes": { - "proxy": { - "sync": { - "rules": { - "duration": { - "us": { - "sum": 763620.9329999998, - "count": 18, - "bucket": { - "1000": 0, - "2000": 0, - "4000": 0, - "8000": 0, - "16000": 0, - "32000": 10, - "64000": 16, - "128000": 17, - "256000": 18, - "512000": 18, - "1024000": 18, - "2048000": 18, - "4096000": 18, - "8192000": 18, - "16384000": 18, - "+Inf": 18 - } - } - } - }, - "networkprogramming": { - "duration": { - "us": { - "count": 19, - "bucket": { - "0": 0, - "250000": 4, - "500000": 8, - "1000000": 11, - "2000000": 11, - "3000000": 11, - "4000000": 11, - "5000000": 11, - "6000000": 11, - "7000000": 11, - "8000000": 11, - "9000000": 11, - "10000000": 11, - "11000000": 11, - "12000000": 11, - "13000000": 11, - "14000000": 11, - "15000000": 11, - "16000000": 11, - "17000000": 11, - "18000000": 11, - "19000000": 11, - "20000000": 11, - "21000000": 11, - "22000000": 11, - "23000000": 11, - "24000000": 11, - "25000000": 11, - "26000000": 11, - "27000000": 11, - "28000000": 11, - "29000000": 11, - "30000000": 11, - "31000000": 11, - "32000000": 11, - "33000000": 11, - "34000000": 11, - "35000000": 11, - "36000000": 11, - "37000000": 11, - "38000000": 11, - "39000000": 11, - "40000000": 11, - "41000000": 11, - "42000000": 11, - "43000000": 11, - "44000000": 11, - "45000000": 11, - "46000000": 11, - "47000000": 11, - "48000000": 11, - "49000000": 11, - "50000000": 11, - "51000000": 11, - "52000000": 11, - "53000000": 11, - "54000000": 11, - "55000000": 11, - "56000000": 11, - "57000000": 11, - "58000000": 11, - "59000000": 11, - "60000000": 11, - "65000000": 11, - "70000000": 11, - "75000000": 11, - "80000000": 11, - "85000000": 11, - "90000000": 11, - "95000000": 11, - "100000000": 11, - "105000000": 11, - "110000000": 11, - "115000000": 11, - "120000000": 11, - "150000000": 11, - "180000000": 11, - "210000000": 11, - "240000000": 11, - "270000000": 11, - "300000000": 11, - "+Inf": 19 - }, - "sum": 5571080914163.27 + "@timestamp": "2020-06-25T12:35:29.639Z", + "agent": { + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.proxy" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.proxy", + "namespace": "default" + }, + "host": { + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "name": "minikube", + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false + }, + "kubernetes": { + "proxy": { + "sync": { + "rules": { + "duration": { + "us": { + "sum": 763620.9329999998, + "count": 18, + "bucket": { + "1000": 0, + "2000": 0, + "4000": 0, + "8000": 0, + "16000": 0, + "32000": 10, + "64000": 16, + "128000": 17, + "256000": 18, + "512000": 18, + "1024000": 18, + "2048000": 18, + "4096000": 18, + "8192000": 18, + "16384000": 18, + "+Inf": 18 } } } }, - "process": { - "cpu": { - "sec": 8 - }, - "memory": { - "resident": { - "bytes": 37609472 - }, - "virtual": { - "bytes": 143990784 + "networkprogramming": { + "duration": { + "us": { + "count": 19, + "bucket": { + "0": 0, + "250000": 4, + "500000": 8, + "1000000": 11, + "2000000": 11, + "3000000": 11, + "4000000": 11, + "5000000": 11, + "6000000": 11, + "7000000": 11, + "8000000": 11, + "9000000": 11, + "10000000": 11, + "11000000": 11, + "12000000": 11, + "13000000": 11, + "14000000": 11, + "15000000": 11, + "16000000": 11, + "17000000": 11, + "18000000": 11, + "19000000": 11, + "20000000": 11, + "21000000": 11, + "22000000": 11, + "23000000": 11, + "24000000": 11, + "25000000": 11, + "26000000": 11, + "27000000": 11, + "28000000": 11, + "29000000": 11, + "30000000": 11, + "31000000": 11, + "32000000": 11, + "33000000": 11, + "34000000": 11, + "35000000": 11, + "36000000": 11, + "37000000": 11, + "38000000": 11, + "39000000": 11, + "40000000": 11, + "41000000": 11, + "42000000": 11, + "43000000": 11, + "44000000": 11, + "45000000": 11, + "46000000": 11, + "47000000": 11, + "48000000": 11, + "49000000": 11, + "50000000": 11, + "51000000": 11, + "52000000": 11, + "53000000": 11, + "54000000": 11, + "55000000": 11, + "56000000": 11, + "57000000": 11, + "58000000": 11, + "59000000": 11, + "60000000": 11, + "65000000": 11, + "70000000": 11, + "75000000": 11, + "80000000": 11, + "85000000": 11, + "90000000": 11, + "95000000": 11, + "100000000": 11, + "105000000": 11, + "110000000": 11, + "115000000": 11, + "120000000": 11, + "150000000": 11, + "180000000": 11, + "210000000": 11, + "240000000": 11, + "270000000": 11, + "300000000": 11, + "+Inf": 19 + }, + "sum": 5571080914163.27 } + } + } + }, + "process": { + "cpu": { + "sec": 8 + }, + "memory": { + "resident": { + "bytes": 37609472 }, - "started": { - "sec": 1593069580.69 - }, - "fds": { - "open": { - "count": 17 - } + "virtual": { + "bytes": 143990784 + } + }, + "started": { + "sec": 1593069580.69 + }, + "fds": { + "open": { + "count": 17 } } } - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "kubernetes", - "duration": 2031254, - "dataset": "kubernetes.proxy" - }, - "metricset": { - "name": "proxy", - "period": 10000 - }, - "service": { - "address": "localhost:10249", - "type": "kubernetes" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:35:29.639Z" - ] + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1593088529639 - ] + "event": { + "module": "kubernetes", + "duration": 2031254, + "dataset": "kubernetes.proxy" + }, + "metricset": { + "name": "proxy", + "period": 10000 + }, + "service": { + "address": "localhost:10249", + "type": "kubernetes" + } } ``` @@ -1751,107 +1635,93 @@ An example event for `scheduler` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.scheduler-default-000001", - "_id": "01l663IBolOt49UrTW36", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:35:59.624Z", - "agent": { - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube", - "type": "metricbeat" - }, - "dataset": { - "name": "kubernetes.scheduler", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.scheduler" - }, - "host": { - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" - }, - "name": "minikube", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ] - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "duration": 7245648, - "dataset": "kubernetes.scheduler", - "module": "kubernetes" - }, - "metricset": { - "name": "scheduler", - "period": 10000 - }, - "service": { - "address": "localhost:10251", - "type": "kubernetes" - }, - "kubernetes": { - "scheduler": { - "name": "kube-scheduler", - "leader": { - "is_master": true - } - } - } + "@timestamp": "2020-06-25T12:35:59.624Z", + "agent": { + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat" + }, + "dataset": { + "name": "kubernetes.scheduler", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.scheduler" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:35:59.624Z" + "host": { + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "name": "minikube", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" ] }, - "sort": [ - 1593088559624 - ] + "ecs": { + "version": "1.5.0" + }, + "event": { + "duration": 7245648, + "dataset": "kubernetes.scheduler", + "module": "kubernetes" + }, + "metricset": { + "name": "scheduler", + "period": 10000 + }, + "service": { + "address": "localhost:10251", + "type": "kubernetes" + }, + "kubernetes": { + "scheduler": { + "name": "kube-scheduler", + "leader": { + "is_master": true + } + } + } } ``` @@ -1940,108 +1810,94 @@ An example event for `state_container` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_container-default-000001", - "_id": "P1l663IBolOt49Ur1YbF", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:36:34.469Z", - "host": { - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false + "@timestamp": "2020-06-25T12:36:34.469Z", + "host": { + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false + }, + "event": { + "dataset": "kubernetes.container", + "module": "kubernetes", + "duration": 8554499 + }, + "kubernetes": { + "node": { + "name": "minikube" }, - "event": { - "dataset": "kubernetes.container", - "module": "kubernetes", - "duration": 8554499 + "labels": { + "component": "kube-scheduler", + "tier": "control-plane" }, - "kubernetes": { - "node": { - "name": "minikube" - }, - "labels": { - "component": "kube-scheduler", - "tier": "control-plane" - }, - "container": { - "image": "k8s.gcr.io/kube-scheduler:v1.17.0", - "name": "kube-scheduler", - "cpu": { - "request": { - "cores": 0.1 - } - }, - "status": { - "phase": "running", - "ready": true, - "restarts": 10 - }, - "id": "docker://b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" + "container": { + "image": "k8s.gcr.io/kube-scheduler:v1.17.0", + "name": "kube-scheduler", + "cpu": { + "request": { + "cores": 0.1 + } }, - "pod": { - "name": "kube-scheduler-minikube", - "uid": "9cdbd5ea-7638-4e86-a706-a5b222d86f26" + "status": { + "phase": "running", + "ready": true, + "restarts": 10 }, - "namespace": "kube-system" - }, - "dataset": { - "name": "kubernetes.state_container", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_container", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" - }, - "container": { - "runtime": "docker", - "id": "b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" + "id": "docker://b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" + "pod": { + "name": "kube-scheduler-minikube", + "uid": "9cdbd5ea-7638-4e86-a706-a5b222d86f26" }, - "metricset": { - "name": "state_container", - "period": 10000 - } + "namespace": "kube-system" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:36:34.469Z" - ] + "dataset": { + "name": "kubernetes.state_container", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_container", + "namespace": "default" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" }, - "sort": [ - 1593088594469 - ] + "container": { + "runtime": "docker", + "id": "b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "metricset": { + "name": "state_container", + "period": 10000 + } } ``` @@ -2120,94 +1976,80 @@ An example event for `state_cronjob` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_cronjob-default-000001", - "_id": "qFqA63IBolOt49Urybs0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:43:04.384Z", - "metricset": { - "name": "state_cronjob", - "period": 10000 - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)" - } - }, - "event": { - "dataset": "kubernetes.cronjob", - "module": "kubernetes", - "duration": 9482053 - }, - "kubernetes": { - "namespace": "default", - "cronjob": { - "active": { - "count": 0 - }, - "is_suspended": false, - "name": "hello", - "next_schedule": { - "sec": 1593088980 - }, - "last_schedule": { - "sec": 1593088920 - }, - "created": { - "sec": 1593088862 - } + "@timestamp": "2020-06-25T12:43:04.384Z", + "metricset": { + "name": "state_cronjob", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + }, + "event": { + "dataset": "kubernetes.cronjob", + "module": "kubernetes", + "duration": 9482053 + }, + "kubernetes": { + "namespace": "default", + "cronjob": { + "active": { + "count": 0 + }, + "is_suspended": false, + "name": "hello", + "next_schedule": { + "sec": 1593088980 + }, + "last_schedule": { + "sec": 1593088920 + }, + "created": { + "sec": 1593088862 } - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_cronjob", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_cronjob" - }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:43:04.384Z" - ] + "dataset": { + "type": "metrics", + "name": "kubernetes.state_cronjob", + "namespace": "default" }, - "sort": [ - 1593088984384 - ] + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.state_cronjob" + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + } } ``` @@ -2283,90 +2125,76 @@ An example event for `state_daemonset` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_daemonset-default-000001", - "_id": "H1l763IBolOt49UrSp72", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:37:04.455Z", - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "event": { - "module": "kubernetes", - "duration": 8648138, - "dataset": "kubernetes.daemonset" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "name": "state_daemonset", - "period": 10000 - }, - "kubernetes": { - "daemonset": { - "name": "metricbeat", - "replicas": { - "available": 1, - "desired": 1, - "ready": 1, - "unavailable": 0 - } - }, - "labels": { - "k8s-app": "metricbeat" - }, - "namespace": "kube-system" - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_daemonset", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_daemonset", - "namespace": "default" + "@timestamp": "2020-06-25T12:37:04.455Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "module": "kubernetes", + "duration": 8648138, + "dataset": "kubernetes.daemonset" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "state_daemonset", + "period": 10000 + }, + "kubernetes": { + "daemonset": { + "name": "metricbeat", + "replicas": { + "available": 1, + "desired": 1, + "ready": 1, + "unavailable": 0 + } }, - "host": { - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat" - }, - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ] + "labels": { + "k8s-app": "metricbeat" }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - } + "namespace": "kube-system" + }, + "dataset": { + "type": "metrics", + "name": "kubernetes.state_daemonset", + "namespace": "default" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:37:04.455Z" + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_daemonset", + "namespace": "default" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat" + }, + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ] }, - "sort": [ - 1593088624455 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + } } ``` @@ -2438,91 +2266,77 @@ An example event for `state_deployment` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_deployment-default-000001", - "_id": "H1l763IBolOt49UrSp72", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:37:04.455Z", - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "event": { - "module": "kubernetes", - "duration": 8648138, - "dataset": "kubernetes.deployment" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "name": "state_deployment", - "period": 10000 - }, - "kubernetes": { - "deployment": { - "name": "metricbeat", - "replicas": { - "unavailable": 0, - "desired": 1, - "updated": 1, - "available": 1 - }, - "paused": false - }, - "labels": { - "k8s-app": "metricbeat" + "@timestamp": "2020-06-25T12:37:04.455Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "module": "kubernetes", + "duration": 8648138, + "dataset": "kubernetes.deployment" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "state_deployment", + "period": 10000 + }, + "kubernetes": { + "deployment": { + "name": "metricbeat", + "replicas": { + "unavailable": 0, + "desired": 1, + "updated": 1, + "available": 1 }, - "namespace": "kube-system" + "paused": false }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_deployment", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_deployment", - "namespace": "default" - }, - "host": { - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat" - }, - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ] + "labels": { + "k8s-app": "metricbeat" }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - } + "namespace": "kube-system" + }, + "dataset": { + "type": "metrics", + "name": "kubernetes.state_deployment", + "namespace": "default" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_deployment", + "namespace": "default" }, - "fields": { - "@timestamp": [ - "2020-06-25T12:37:04.455Z" + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat" + }, + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ] }, - "sort": [ - 1593088624455 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + } } ``` @@ -2594,116 +2408,102 @@ An example event for `state_node` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_node-default-000001", - "_id": "c1l763IBolOt49Ur58c8", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:37:44.457Z", - "ecs": { - "version": "1.5.0" - }, - "host": { - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ] - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_node" - }, - "metricset": { - "name": "state_node", - "period": 10000 - }, - "kubernetes": { - "node": { - "pod": { - "capacity": { - "total": 110 - }, - "allocatable": { - "total": 110 - } - }, - "memory": { - "capacity": { - "bytes": 16815325184 - }, - "allocatable": { - "bytes": 16815325184 - } + "@timestamp": "2020-06-25T12:37:44.457Z", + "ecs": { + "version": "1.5.0" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ] + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.state_node" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.state_node" + }, + "metricset": { + "name": "state_node", + "period": 10000 + }, + "kubernetes": { + "node": { + "pod": { + "capacity": { + "total": 110 }, - "cpu": { - "allocatable": { - "cores": 4 - }, - "capacity": { - "cores": 4 - } + "allocatable": { + "total": 110 + } + }, + "memory": { + "capacity": { + "bytes": 16815325184 + }, + "allocatable": { + "bytes": 16815325184 + } + }, + "cpu": { + "allocatable": { + "cores": 4 }, - "name": "minikube", - "status": { - "ready": "true", - "unschedulable": false + "capacity": { + "cores": 4 } }, - "labels": { - "kubernetes_io/arch": "amd64", - "kubernetes_io/hostname": "minikube", - "kubernetes_io/os": "linux", - "node-role_kubernetes_io/master": "", - "beta_kubernetes_io/arch": "amd64", - "beta_kubernetes_io/os": "linux" + "name": "minikube", + "status": { + "ready": "true", + "unschedulable": false } }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" - }, - "service": { - "type": "kubernetes", - "address": "kube-state-metrics:8080" - }, - "event": { - "dataset": "kubernetes.node", - "module": "kubernetes", - "duration": 8194220 + "labels": { + "kubernetes_io/arch": "amd64", + "kubernetes_io/hostname": "minikube", + "kubernetes_io/os": "linux", + "node-role_kubernetes_io/master": "", + "beta_kubernetes_io/arch": "amd64", + "beta_kubernetes_io/os": "linux" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:37:44.457Z" - ] + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" }, - "sort": [ - 1593088664457 - ] + "service": { + "type": "kubernetes", + "address": "kube-state-metrics:8080" + }, + "event": { + "dataset": "kubernetes.node", + "module": "kubernetes", + "duration": 8194220 + } } ``` @@ -2782,88 +2582,74 @@ An example event for `state_persistentvolume` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_persistentvolume-default-000001", - "_id": "8lqB63IBolOt49UrjOyD", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:43:54.412Z", - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "kubernetes", - "duration": 12149615, - "dataset": "kubernetes.persistentvolume" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat" - }, - "kubernetes": { - "persistentvolume": { - "capacity": { - "bytes": 10737418240 - }, - "phase": "Bound", - "storage_class": "manual", - "name": "task-pv-volume" - }, - "labels": { - "type": "local" - } - }, - "dataset": { - "name": "kubernetes.state_persistentvolume", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolume", - "namespace": "default" - }, - "host": { - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" + "@timestamp": "2020-06-25T12:43:54.412Z", + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "kubernetes", + "duration": 12149615, + "dataset": "kubernetes.persistentvolume" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + }, + "kubernetes": { + "persistentvolume": { + "capacity": { + "bytes": 10737418240 }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "containerized": false - }, - "metricset": { - "period": 10000, - "name": "state_persistentvolume" + "phase": "Bound", + "storage_class": "manual", + "name": "task-pv-volume" }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" + "labels": { + "type": "local" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:43:54.412Z" - ] + "dataset": { + "name": "kubernetes.state_persistentvolume", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_persistentvolume", + "namespace": "default" + }, + "host": { + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false }, - "sort": [ - 1593089034412 - ] + "metricset": { + "period": 10000, + "name": "state_persistentvolume" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + } } ``` @@ -2934,88 +2720,74 @@ An example event for `state_persistentvolumeclaim` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_persistentvolumeclaim-default-000001", - "_id": "6FuC63IBolOt49UrTxrR", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:44:44.418Z", - "event": { - "dataset": "kubernetes.persistentvolumeclaim", - "module": "kubernetes", - "duration": 5698588 - }, - "metricset": { - "name": "state_persistentvolumeclaim", - "period": 10000 - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "kubernetes": { - "namespace": "default", - "persistentvolumeclaim": { - "phase": "Bound", - "storage_class": "manual", - "volume_name": "task-pv-volume", - "name": "task-pv-claim", - "request_storage": { - "bytes": 3221225472 - }, - "access_mode": "ReadWriteOnce" - } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_persistentvolumeclaim" - }, - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolumeclaim", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" + "@timestamp": "2020-06-25T12:44:44.418Z", + "event": { + "dataset": "kubernetes.persistentvolumeclaim", + "module": "kubernetes", + "duration": 5698588 + }, + "metricset": { + "name": "state_persistentvolumeclaim", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "kubernetes": { + "namespace": "default", + "persistentvolumeclaim": { + "phase": "Bound", + "storage_class": "manual", + "volume_name": "task-pv-volume", + "name": "task-pv-claim", + "request_storage": { + "bytes": 3221225472 }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64" + "access_mode": "ReadWriteOnce" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:44:44.418Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.state_persistentvolumeclaim" + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" }, - "sort": [ - 1593089084418 - ] + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_persistentvolumeclaim", + "namespace": "default" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64" + } } ``` @@ -3088,97 +2860,83 @@ An example event for `state_pod` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_pod-default-000001", - "_id": "YVl863IBolOt49UrqueH", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:38:34.469Z", - "dataset": { - "name": "kubernetes.state_pod", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_pod" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" - } - }, - "event": { - "duration": 10777415, - "dataset": "kubernetes.pod", - "module": "kubernetes" - }, - "service": { - "type": "kubernetes", - "address": "kube-state-metrics:8080" - }, - "kubernetes": { - "pod": { - "name": "filebeat-dqzzz", - "status": { - "ready": "true", - "scheduled": "true", - "phase": "running" - }, - "host_ip": "192.168.64.10", - "ip": "192.168.64.10", - "uid": "a5f1d3c9-40b6-4182-823b-dd5ff9832279" - }, - "namespace": "kube-system", - "node": { - "name": "minikube" + "@timestamp": "2020-06-25T12:38:34.469Z", + "dataset": { + "name": "kubernetes.state_pod", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.state_pod" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + } + }, + "event": { + "duration": 10777415, + "dataset": "kubernetes.pod", + "module": "kubernetes" + }, + "service": { + "type": "kubernetes", + "address": "kube-state-metrics:8080" + }, + "kubernetes": { + "pod": { + "name": "filebeat-dqzzz", + "status": { + "ready": "true", + "scheduled": "true", + "phase": "running" }, - "labels": { - "controller-revision-hash": "85649b9ddb", - "k8s-app": "filebeat", - "pod-template-generation": "1" - } + "host_ip": "192.168.64.10", + "ip": "192.168.64.10", + "uid": "a5f1d3c9-40b6-4182-823b-dd5ff9832279" }, - "agent": { - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" + "namespace": "kube-system", + "node": { + "name": "minikube" }, - "metricset": { - "period": 10000, - "name": "state_pod" + "labels": { + "controller-revision-hash": "85649b9ddb", + "k8s-app": "filebeat", + "pod-template-generation": "1" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:38:34.469Z" - ] + "agent": { + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" }, - "sort": [ - 1593088714469 - ] + "metricset": { + "period": 10000, + "name": "state_pod" + } } ``` @@ -3251,97 +3009,83 @@ An example event for `state_replicaset` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_replicaset-default-000001", - "_id": "U1l863IBolOt49Ur-Pu2", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:38:54.482Z", - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "metricset": { - "period": 10000, - "name": "state_replicaset" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_replicaset" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_replicaset", - "namespace": "default" - }, - "event": { - "module": "kubernetes", - "duration": 5456128, - "dataset": "kubernetes.replicaset" - }, - "kubernetes": { - "namespace": "kube-system", - "replicaset": { - "name": "nginx-ingress-controller-6fc5bcc8c9", - "replicas": { - "labeled": 1, - "ready": 1, - "available": 1, - "observed": 1, - "desired": 1 - } - }, - "deployment": { - "name": "nginx-ingress-controller" - }, - "labels": { - "app_kubernetes_io/part-of": "kube-system", - "pod-template-hash": "6fc5bcc8c9", - "addonmanager_kubernetes_io/mode": "Reconcile", - "app_kubernetes_io/name": "nginx-ingress-controller" + "@timestamp": "2020-06-25T12:38:54.482Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "metricset": { + "period": 10000, + "name": "state_replicaset" + }, + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.state_replicaset" + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_replicaset", + "namespace": "default" + }, + "event": { + "module": "kubernetes", + "duration": 5456128, + "dataset": "kubernetes.replicaset" + }, + "kubernetes": { + "namespace": "kube-system", + "replicaset": { + "name": "nginx-ingress-controller-6fc5bcc8c9", + "replicas": { + "labeled": 1, + "ready": 1, + "available": 1, + "observed": 1, + "desired": 1 } }, - "agent": { - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0" - }, - "ecs": { - "version": "1.5.0" + "deployment": { + "name": "nginx-ingress-controller" }, - "host": { - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - }, - "id": "b0e83d397c054b8a99a431072fe4617b" + "labels": { + "app_kubernetes_io/part-of": "kube-system", + "pod-template-hash": "6fc5bcc8c9", + "addonmanager_kubernetes_io/mode": "Reconcile", + "app_kubernetes_io/name": "nginx-ingress-controller" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:38:54.482Z" - ] + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" }, - "sort": [ - 1593088734482 - ] + "ecs": { + "version": "1.5.0" + }, + "host": { + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b" + } } ``` @@ -3413,84 +3157,70 @@ An example event for `state_resourcequota` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_resourcequota-default-000001", - "_id": "4FuC63IBolOt49UrnSHz", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:45:04.416Z", - "metricset": { - "name": "state_resourcequota", - "period": 10000 - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_resourcequota", - "namespace": "default" - }, - "host": { - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ] - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "event": { - "dataset": "kubernetes.resourcequota", - "module": "kubernetes", - "duration": 6324269 - }, - "agent": { - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" - }, - "ecs": { - "version": "1.5.0" - }, - "kubernetes": { - "namespace": "quota-object-example", - "resourcequota": { - "name": "object-quota-demo", - "resource": "persistentvolumeclaims", - "type": "hard", - "quota": 1 - } - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_resourcequota", - "namespace": "default" - } + "@timestamp": "2020-06-25T12:45:04.416Z", + "metricset": { + "name": "state_resourcequota", + "period": 10000 }, - "fields": { - "@timestamp": [ - "2020-06-25T12:45:04.416Z" + "dataset": { + "type": "metrics", + "name": "kubernetes.state_resourcequota", + "namespace": "default" + }, + "host": { + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" ] }, - "sort": [ - 1593089104416 - ] + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.resourcequota", + "module": "kubernetes", + "duration": 6324269 + }, + "agent": { + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" + }, + "ecs": { + "version": "1.5.0" + }, + "kubernetes": { + "namespace": "quota-object-example", + "resourcequota": { + "name": "object-quota-demo", + "resource": "persistentvolumeclaims", + "type": "hard", + "quota": 1 + } + }, + "stream": { + "type": "metrics", + "dataset": "kubernetes.state_resourcequota", + "namespace": "default" + } } ``` @@ -3562,93 +3292,76 @@ An example event for `state_service` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_service-default-000001", - "_id": "Elp963IBolOt49UrbRPd", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:24.389Z", - "kubernetes": { - "labels": { - "kubernetes_io_minikube_addons_endpoint": "metrics-server", - "kubernetes_io_name": "Metrics-server", - "addonmanager_kubernetes_io_mode": "Reconcile", - "kubernetes_io_minikube_addons": "metrics-server" - }, - "service": { - "name": "metrics-server", - "created": "2020-06-10T09:02:27.000Z", - "cluster_ip": "10.96.124.248", - "type": "ClusterIP" - }, - "namespace": "kube-system" - }, - "event": { - "dataset": "kubernetes.service", - "module": "kubernetes", - "duration": 10966648 - }, - "metricset": { - "name": "state_service", - "period": 10000 - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - } - }, - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-06-25T12:39:24.389Z", + "kubernetes": { + "labels": { + "kubernetes_io_minikube_addons_endpoint": "metrics-server", + "kubernetes_io_name": "Metrics-server", + "addonmanager_kubernetes_io_mode": "Reconcile", + "kubernetes_io_minikube_addons": "metrics-server" }, "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" + "name": "metrics-server", + "created": "2020-06-10T09:02:27.000Z", + "cluster_ip": "10.96.124.248", + "type": "ClusterIP" }, - "stream": { - "dataset": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" - } + "namespace": "kube-system" + }, + "event": { + "dataset": "kubernetes.service", + "module": "kubernetes", + "duration": 10966648 }, - "fields": { - "@timestamp": [ - "2020-06-25T12:39:24.389Z" + "metricset": { + "name": "state_service", + "period": 10000 + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.service.created": [ - "2020-06-10T09:02:27.000Z" - ] + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1593088764389 - ] + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "dataset": { + "name": "kubernetes.state_service", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "dataset": "kubernetes.state_service", + "namespace": "default", + "type": "metrics" + } } ``` @@ -3723,93 +3436,76 @@ An example event for `state_statefulset` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_statefulset-default-000001", - "_id": "Elp963IBolOt49UrbRPd", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:24.389Z", - "kubernetes": { - "namespace": "default", - "statefulset": { - "created": 1511989697, - "generation": { - "desired": 4, - "observed": 2 - }, - "name": "mysql", - "replicas": { - "desired": 5, - "observed": 2 - } - } - }, - "event": { - "dataset": "kubernetes.statefulset", - "module": "kubernetes", - "duration": 10966648 - }, - "metricset": { - "name": "state_statefulset", - "period": 10000 - }, - "host": { - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ], - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" + "@timestamp": "2020-06-25T12:39:24.389Z", + "kubernetes": { + "namespace": "default", + "statefulset": { + "created": 1511989697, + "generation": { + "desired": 4, + "observed": 2 + }, + "name": "mysql", + "replicas": { + "desired": 5, + "observed": 2 } - }, - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "ecs": { - "version": "1.5.0" - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:39:24.389Z" + "event": { + "dataset": "kubernetes.statefulset", + "module": "kubernetes", + "duration": 10966648 + }, + "metricset": { + "name": "state_statefulset", + "period": 10000 + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.statefulset.created": [ - "2020-06-10T09:02:27.000Z" - ] + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" }, - "sort": [ - 1593088764389 - ] + "dataset": { + "name": "kubernetes.state_statefulset", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "dataset": "kubernetes.state_statefulset", + "namespace": "default", + "type": "metrics" + } } ``` @@ -3881,90 +3577,73 @@ An example event for `state_storageclass` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.state_storageclass-default-000001", - "_id": "KFp963IBolOt49UruyX3", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:44.399Z", - "agent": { - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", - "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" - }, - "kubernetes": { - "storageclass": { - "provisioner": "k8s.io/minikube-hostpath", - "reclaim_policy": "Delete", - "volume_binding_mode": "Immediate", - "name": "standard", - "created": "2020-06-10T09:02:27.000Z" - }, - "labels": { - "addonmanager_kubernetes_io_mode": "EnsureExists" - } - }, - "dataset": { - "name": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, - "host": { - "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", - "containerized": false, - "ip": [ - "172.17.0.11" - ], - "mac": [ - "02:42:ac:11:00:0b" - ] - }, - "event": { - "module": "kubernetes", - "duration": 5713503, - "dataset": "kubernetes.storageclass" - }, - "metricset": { - "name": "state_storageclass", - "period": 10000 - }, - "service": { - "address": "kube-state-metrics:8080", - "type": "kubernetes" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-06-25T12:39:44.399Z", + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "kubernetes": { + "storageclass": { + "provisioner": "k8s.io/minikube-hostpath", + "reclaim_policy": "Delete", + "volume_binding_mode": "Immediate", + "name": "standard", + "created": "2020-06-10T09:02:27.000Z" + }, + "labels": { + "addonmanager_kubernetes_io_mode": "EnsureExists" } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:39:44.399Z" + "dataset": { + "name": "kubernetes.state_storageclass", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "dataset": "kubernetes.state_storageclass", + "namespace": "default", + "type": "metrics" + }, + "host": { + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false, + "ip": [ + "172.17.0.11" ], - "kubernetes.storageclass.created": [ - "2020-06-10T09:02:27.000Z" + "mac": [ + "02:42:ac:11:00:0b" ] }, - "sort": [ - 1593088784399 - ] + "event": { + "module": "kubernetes", + "duration": 5713503, + "dataset": "kubernetes.storageclass" + }, + "metricset": { + "name": "state_storageclass", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + } } ``` @@ -4036,132 +3715,115 @@ An example event for `system` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.system-default-000001", - "_id": "sVp963IBolOt49Ur9yyT", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:39:59.647Z", - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.system" - }, - "service": { - "address": "minikube:10250", - "type": "kubernetes" - }, - "event": { - "duration": 20012905, - "dataset": "kubernetes.system", - "module": "kubernetes" - }, - "stream": { - "dataset": "kubernetes.system", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube", - "name": "minikube", - "architecture": "x86_64", - "os": { - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ] - }, - "agent": { - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", - "name": "minikube", - "type": "metricbeat", - "version": "8.0.0" + "@timestamp": "2020-06-25T12:39:59.647Z", + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "kubernetes.system" + }, + "service": { + "address": "minikube:10250", + "type": "kubernetes" + }, + "event": { + "duration": 20012905, + "dataset": "kubernetes.system", + "module": "kubernetes" + }, + "stream": { + "dataset": "kubernetes.system", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "name": "minikube", + "architecture": "x86_64", + "os": { + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ] + }, + "agent": { + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0" + }, + "kubernetes": { + "node": { + "name": "minikube" }, - "kubernetes": { - "node": { - "name": "minikube" - }, - "system": { - "container": "runtime", - "cpu": { - "usage": { - "nanocores": 35779815, - "core": { - "ns": 530899961233 - } + "system": { + "container": "runtime", + "cpu": { + "usage": { + "nanocores": 35779815, + "core": { + "ns": 530899961233 } + } + }, + "memory": { + "pagefaults": 12944019, + "majorpagefaults": 99, + "usage": { + "bytes": 198279168 }, - "memory": { - "pagefaults": 12944019, - "majorpagefaults": 99, - "usage": { - "bytes": 198279168 - }, - "workingset": { - "bytes": 178794496 - }, - "rss": { - "bytes": 125259776 - } + "workingset": { + "bytes": 178794496 }, - "start_time": "2020-06-25T07:19:32Z" - } - }, - "metricset": { - "name": "system", - "period": 10000 + "rss": { + "bytes": 125259776 + } + }, + "start_time": "2020-06-25T07:19:32Z" } }, - "fields": { - "kubernetes.system.start_time": [ - "2020-06-25T07:19:32.000Z" - ], - "@timestamp": [ - "2020-06-25T12:39:59.647Z" - ] - }, - "sort": [ - 1593088799647 - ] + "metricset": { + "name": "system", + "period": 10000 + } } ``` @@ -4237,127 +3899,113 @@ An example event for `volume` looks as following: ```$json { - "_index": ".ds-metrics-kubernetes.volume-default-000001", - "_id": "b1p-63IBolOt49UrRT-d", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:40:19.649Z", - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "name": "volume", - "period": 10000 - }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" - }, - "kubernetes": { - "pod": { - "name": "metricbeat-g9fc6" - }, - "volume": { - "name": "config", - "fs": { - "inodes": { - "used": 5, - "free": 9549949, - "count": 9768928 - }, - "available": { - "bytes": 7719858176 - }, - "capacity": { - "bytes": 17361141760 - }, - "used": { - "bytes": 12288 - } + "@timestamp": "2020-06-25T12:40:19.649Z", + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "volume", + "period": 10000 + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "kubernetes": { + "pod": { + "name": "metricbeat-g9fc6" + }, + "volume": { + "name": "config", + "fs": { + "inodes": { + "used": 5, + "free": 9549949, + "count": 9768928 + }, + "available": { + "bytes": 7719858176 + }, + "capacity": { + "bytes": 17361141760 + }, + "used": { + "bytes": 12288 } - }, - "namespace": "kube-system", - "node": { - "name": "minikube" } }, - "dataset": { - "type": "metrics", - "name": "kubernetes.volume", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.volume" - }, - "host": { - "architecture": "x86_64", - "os": { - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.19.81", - "codename": "Core" - }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "name": "minikube", - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ], - "hostname": "minikube" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "namespace": "kube-system", + "node": { "name": "minikube" - }, - "event": { - "dataset": "kubernetes.volume", - "module": "kubernetes", - "duration": 12481688 } }, - "fields": { - "@timestamp": [ - "2020-06-25T12:40:19.649Z" - ] + "dataset": { + "type": "metrics", + "name": "kubernetes.volume", + "namespace": "default" + }, + "stream": { + "namespace": "default", + "type": "metrics", + "dataset": "kubernetes.volume" + }, + "host": { + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "name": "minikube", + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" }, - "sort": [ - 1593088819649 - ] + "event": { + "dataset": "kubernetes.volume", + "module": "kubernetes", + "duration": 12481688 + } } ``` diff --git a/packages/mongodb/data_stream/collstats/sample_event.json b/packages/mongodb/data_stream/collstats/sample_event.json index af714db9b60..f1a8153f7b9 100644 --- a/packages/mongodb/data_stream/collstats/sample_event.json +++ b/packages/mongodb/data_stream/collstats/sample_event.json @@ -1,110 +1,99 @@ { - "_index": ".ds-metrics-mongodb.collstats-default-000001", - "_id": "6hT0AXMB-2lnjH4qREj1", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.459Z", - "metricset": { - "name": "collstats", - "period": 10000 - }, - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "stream": { - "dataset": "mongodb.collstats", - "namespace": "default", - "type": "metrics" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro" - }, - "event": { - "dataset": "mongodb.collstats", - "module": "mongodb", - "duration": 3378520 - }, - "mongodb": { - "collstats": { - "collection": "startup_log", - "commands": { - "count": 0, - "time": { - "us": 0 - } - }, - "db": "local", - "getmore": { - "count": 0, - "time": { - "us": 0 - } - }, - "insert": { - "count": 0, - "time": { - "us": 0 - } - }, - "lock": { - "read": { - "count": 74, - "time": { - "us": 443 - } - }, - "write": { - "count": 1, - "time": { - "us": 8 - } - } - }, - "name": "local.startup_log", - "queries": { - "count": 0, - "time": { - "us": 0 - } - }, - "remove": { - "count": 0, - "time": { - "us": 0 - } - }, - "total": { - "count": 75, + "@timestamp": "2020-06-29T21:20:51.459Z", + "metricset": { + "name": "collstats", + "period": 10000 + }, + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "stream": { + "dataset": "mongodb.collstats", + "namespace": "default", + "type": "metrics" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro" + }, + "event": { + "dataset": "mongodb.collstats", + "module": "mongodb", + "duration": 3378520 + }, + "mongodb": { + "collstats": { + "collection": "startup_log", + "commands": { + "count": 0, + "time": { + "us": 0 + } + }, + "db": "local", + "getmore": { + "count": 0, + "time": { + "us": 0 + } + }, + "insert": { + "count": 0, + "time": { + "us": 0 + } + }, + "lock": { + "read": { + "count": 74, "time": { - "us": 451 + "us": 443 } }, - "update": { - "count": 0, + "write": { + "count": 1, "time": { - "us": 0 + "us": 8 } } + }, + "name": "local.startup_log", + "queries": { + "count": 0, + "time": { + "us": 0 + } + }, + "remove": { + "count": 0, + "time": { + "us": 0 + } + }, + "total": { + "count": 75, + "time": { + "us": 451 + } + }, + "update": { + "count": 0, + "time": { + "us": 0 + } } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.collstats" - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.459Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "mongodb.collstats" + }, + "ecs": { + "version": "1.5.0" } } \ No newline at end of file diff --git a/packages/mongodb/data_stream/dbstats/sample_event.json b/packages/mongodb/data_stream/dbstats/sample_event.json index 46ece42e5f0..a6926020b78 100644 --- a/packages/mongodb/data_stream/dbstats/sample_event.json +++ b/packages/mongodb/data_stream/dbstats/sample_event.json @@ -1,70 +1,59 @@ { - "_index": ".ds-metrics-mongodb.dbstats-default-000001", - "_id": "6hT0AXMB-2lnjH4qREj0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.459Z", - "metricset": { - "name": "dbstats", - "period": 10000 - }, - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "stream": { - "dataset": "mongodb.dbstats", - "namespace": "default", - "type": "metrics" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro" - }, - "event": { - "dataset": "mongodb.dbstats", - "module": "mongodb", - "duration": 3378520 - }, - "mongodb": { - "dbstats": { - "file_size": {}, - "index_size": { - "bytes": 20480 - }, - "ns_size_mb": {}, - "storage_size": { - "bytes": 20480 - }, - "num_extents": 0, - "collections": 1, - "objects": 1, - "db": "admin", - "data_size": { - "bytes": 59 - }, - "indexes": 1, - "avg_obj_size": { - "bytes": 59 - } + "@timestamp": "2020-06-29T21:20:51.459Z", + "metricset": { + "name": "dbstats", + "period": 10000 + }, + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "stream": { + "dataset": "mongodb.dbstats", + "namespace": "default", + "type": "metrics" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro" + }, + "event": { + "dataset": "mongodb.dbstats", + "module": "mongodb", + "duration": 3378520 + }, + "mongodb": { + "dbstats": { + "file_size": {}, + "index_size": { + "bytes": 20480 + }, + "ns_size_mb": {}, + "storage_size": { + "bytes": 20480 + }, + "num_extents": 0, + "collections": 1, + "objects": 1, + "db": "admin", + "data_size": { + "bytes": 59 + }, + "indexes": 1, + "avg_obj_size": { + "bytes": 59 } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.dbstats" - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.459Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "mongodb.dbstats" + }, + "ecs": { + "version": "1.5.0" } } \ No newline at end of file diff --git a/packages/mongodb/data_stream/log/sample_event.json b/packages/mongodb/data_stream/log/sample_event.json index 0281d98f5bb..645b270356e 100644 --- a/packages/mongodb/data_stream/log/sample_event.json +++ b/packages/mongodb/data_stream/log/sample_event.json @@ -1,70 +1,53 @@ { - "_index": ".ds-logs-mongodb.log-default-000001", - "_id": "BBTwAXMB-2lnjH4q7Sxs", - "_version": 1, - "_score": null, - "_source": { - "container": { - "id": "mongodb" - }, - "agent": { - "name": "KaiyanMacBookPro", - "id": "8e0c8cfc-69bd-4f15-a2e0-7c6ed1f0963a", - "ephemeral_id": "a82d1c90-90b8-44e6-94ac-d0ca900b4948", - "type": "filebeat", - "version": "8.0.0" - }, - "log": { - "file": { - "path": "/usr/local/var/log/mongodb/mongo.log" - }, - "offset": 584519, - "level": "I" - }, - "message": "end connection 127.0.0.1:60764 (1 connection now open)", - "input": { - "type": "log" - }, - "@timestamp": "2020-06-29T21:17:11.459Z", - "ecs": { - "version": "1.5.0" - }, - "stream": { - "namespace": "default", - "type": "logs", - "dataset": "mongodb.log" - }, - "event": { - "created": "2020-06-29T21:17:12.442Z", - "kind": "event", - "category": [ - "database" - ], - "type": [ - "info" - ] - }, - "dataset": { - "namespace": "default", - "name": "mongodb.log", - "type": "logs" + "container": { + "id": "mongodb" + }, + "agent": { + "name": "KaiyanMacBookPro", + "id": "8e0c8cfc-69bd-4f15-a2e0-7c6ed1f0963a", + "ephemeral_id": "a82d1c90-90b8-44e6-94ac-d0ca900b4948", + "type": "filebeat", + "version": "8.0.0" + }, + "log": { + "file": { + "path": "/usr/local/var/log/mongodb/mongo.log" }, - "mongodb": { - "log": { - "component": "NETWORK", - "context": "conn2180" - } - } + "offset": 584519, + "level": "I" + }, + "message": "end connection 127.0.0.1:60764 (1 connection now open)", + "input": { + "type": "log" + }, + "@timestamp": "2020-06-29T21:17:11.459Z", + "ecs": { + "version": "1.5.0" }, - "fields": { - "@timestamp": [ - "2020-06-29T21:17:11.459Z" + "stream": { + "namespace": "default", + "type": "logs", + "dataset": "mongodb.log" + }, + "event": { + "created": "2020-06-29T21:17:12.442Z", + "kind": "event", + "category": [ + "database" ], - "event.created": [ - "2020-06-29T21:17:12.442Z" + "type": [ + "info" ] }, - "sort": [ - 1593465431459 - ] + "dataset": { + "namespace": "default", + "name": "mongodb.log", + "type": "logs" + }, + "mongodb": { + "log": { + "component": "NETWORK", + "context": "conn2180" + } + } } \ No newline at end of file diff --git a/packages/mongodb/data_stream/metrics/sample_event.json b/packages/mongodb/data_stream/metrics/sample_event.json index a8e5e154fe5..8a628a45e8c 100644 --- a/packages/mongodb/data_stream/metrics/sample_event.json +++ b/packages/mongodb/data_stream/metrics/sample_event.json @@ -1,262 +1,251 @@ { - "_index": ".ds-metrics-mongodb.metrics-default-000001", - "_id": "6RT0AXMB-2lnjH4qREj0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.459Z", - "mongodb": { - "metrics": { - "replication": { - "network": { - "ops": 0, - "reders_created": 0, - "bytes": 0, - "getmores": { - "count": 0, - "time": { - "ms": 0 - } + "@timestamp": "2020-06-29T21:20:51.459Z", + "mongodb": { + "metrics": { + "replication": { + "network": { + "ops": 0, + "reders_created": 0, + "bytes": 0, + "getmores": { + "count": 0, + "time": { + "ms": 0 } - }, - "executor": { - "shutting_down": false, - "network_interface": "DEPRECATED: getDiagnosticString is deprecated in NetworkInterfaceTL", - "queues": { - "in_progress": { - "network": 0 - }, - "sleepers": 0 - }, - "unsignaled_events": 0 - }, - "apply": { - "attempts_to_become_secondary": 0, - "batches": { - "count": 0, - "time": { - "ms": 0 - } - }, - "ops": 0 - }, - "buffer": { - "max_size": { - "bytes": 0 - }, - "size": { - "bytes": 0 - }, - "count": 0 - }, - "initial_sync": { - "completed": 0, - "failed_attempts": 0, - "failures": 0 } }, - "ttl": { - "passes": { - "count": 433 + "executor": { + "shutting_down": false, + "network_interface": "DEPRECATED: getDiagnosticString is deprecated in NetworkInterfaceTL", + "queues": { + "in_progress": { + "network": 0 + }, + "sleepers": 0 }, - "deleted_documents": { - "count": 3 - } + "unsignaled_events": 0 }, - "commands": { - "replset_heartbeat": { - "failed": 0, - "total": 0 - }, - "connection_pool_stats": { - "failed": 0, - "total": 0 - }, - "host_info": { - "failed": 0, - "total": 0 - }, - "aggregate": { - "failed": 0, - "total": 0 - }, - "replset_update_position": { - "total": 0, - "failed": 0 - }, - "last_collections": { - "failed": 0, - "total": 458 - }, - "list_databased": { - "total": 466, - "failed": 0 - }, - "whatsmyuri": { - "total": 2, - "failed": 0 - }, - "profile": { - "failed": 0, - "total": 0 - }, - "insert": { - "failed": 0, - "total": 7 - }, - "count": { - "failed": 0, - "total": 0 - }, - "is_master": { - "failed": 0, - "total": 2332 - }, - "distinct": { - "failed": 0, - "total": 0 - }, - "replset_get_status": { - "failed": 2, - "total": 2 - }, - "find": { - "failed": 0, - "total": 94 - }, - "replset_get_rbid": { - "failed": 0, - "total": 0 - }, - "get_parameter": { - "failed": 0, - "total": 0 - }, - "coll_stats": { - "failed": 0, - "total": 0 - }, - "build_info": { - "total": 6, - "failed": 0 - }, - "last_commands": { - "failed": 0, - "total": 0 - }, - "update": { - "failed": 0, - "total": 5 - }, - "is_self": { - "failed": 0, - "total": 0 - }, - "db_stats": { - "failed": 0, - "total": 2044 - }, - "get_cmd_line_opts": { - "failed": 0, - "total": 2 - }, - "ping": { - "total": 2290, - "failed": 0 - }, - "server_status": { - "total": 916, - "failed": 0 - }, - "get_last_error": { - "failed": 0, - "total": 0 + "apply": { + "attempts_to_become_secondary": 0, + "batches": { + "count": 0, + "time": { + "ms": 0 + } }, - "get_more": { - "failed": 0, - "total": 0 + "ops": 0 + }, + "buffer": { + "max_size": { + "bytes": 0 }, - "get_log": { - "failed": 0, - "total": 2 + "size": { + "bytes": 0 }, - "list_indexes": { - "failed": 0, - "total": 174 - } + "count": 0 }, - "cursor": { - "timed_out": 0, - "open": { - "pinned": 0, - "total": 0, - "no_timeout": 0 - } + "initial_sync": { + "completed": 0, + "failed_attempts": 0, + "failures": 0 + } + }, + "ttl": { + "passes": { + "count": 433 + }, + "deleted_documents": { + "count": 3 + } + }, + "commands": { + "replset_heartbeat": { + "failed": 0, + "total": 0 + }, + "connection_pool_stats": { + "failed": 0, + "total": 0 + }, + "host_info": { + "failed": 0, + "total": 0 + }, + "aggregate": { + "failed": 0, + "total": 0 + }, + "replset_update_position": { + "total": 0, + "failed": 0 + }, + "last_collections": { + "failed": 0, + "total": 458 + }, + "list_databased": { + "total": 466, + "failed": 0 + }, + "whatsmyuri": { + "total": 2, + "failed": 0 + }, + "profile": { + "failed": 0, + "total": 0 + }, + "insert": { + "failed": 0, + "total": 7 + }, + "count": { + "failed": 0, + "total": 0 + }, + "is_master": { + "failed": 0, + "total": 2332 + }, + "distinct": { + "failed": 0, + "total": 0 + }, + "replset_get_status": { + "failed": 2, + "total": 2 + }, + "find": { + "failed": 0, + "total": 94 + }, + "replset_get_rbid": { + "failed": 0, + "total": 0 + }, + "get_parameter": { + "failed": 0, + "total": 0 + }, + "coll_stats": { + "failed": 0, + "total": 0 + }, + "build_info": { + "total": 6, + "failed": 0 + }, + "last_commands": { + "failed": 0, + "total": 0 + }, + "update": { + "failed": 0, + "total": 5 + }, + "is_self": { + "failed": 0, + "total": 0 + }, + "db_stats": { + "failed": 0, + "total": 2044 + }, + "get_cmd_line_opts": { + "failed": 0, + "total": 2 + }, + "ping": { + "total": 2290, + "failed": 0 + }, + "server_status": { + "total": 916, + "failed": 0 }, "get_last_error": { - "write_wait": { - "ms": 0, - "count": 0 - }, - "write_timeouts": 0 + "failed": 0, + "total": 0 }, - "operation": { - "write_conflicts": 0, - "scan_and_order": 0 + "get_more": { + "failed": 0, + "total": 0 }, - "document": { - "deleted": 15, - "inserted": 19, - "returned": 465, - "updated": 2 + "get_log": { + "failed": 0, + "total": 2 }, - "query_executor": { - "scanned_indexes": { - "count": 2 - }, - "scanned_documents": { - "count": 24 - } + "list_indexes": { + "failed": 0, + "total": 174 + } + }, + "cursor": { + "timed_out": 0, + "open": { + "pinned": 0, + "total": 0, + "no_timeout": 0 + } + }, + "get_last_error": { + "write_wait": { + "ms": 0, + "count": 0 + }, + "write_timeouts": 0 + }, + "operation": { + "write_conflicts": 0, + "scan_and_order": 0 + }, + "document": { + "deleted": 15, + "inserted": 19, + "returned": 465, + "updated": 2 + }, + "query_executor": { + "scanned_indexes": { + "count": 2 + }, + "scanned_documents": { + "count": 24 } } - }, - "metricset": { - "period": 10000, - "name": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "mongodb.metrics", - "namespace": "default" - }, - "agent": { - "name": "KaiyanMacBookPro", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a" - }, - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "event": { - "dataset": "mongodb.metrics", - "module": "mongodb", - "duration": 3039885 - }, - "dataset": { - "type": "metrics", - "name": "mongodb.metrics", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.459Z" - ] + "metricset": { + "period": 10000, + "name": "metrics" + }, + "stream": { + "type": "metrics", + "dataset": "mongodb.metrics", + "namespace": "default" + }, + "agent": { + "name": "KaiyanMacBookPro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a" + }, + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "event": { + "dataset": "mongodb.metrics", + "module": "mongodb", + "duration": 3039885 + }, + "dataset": { + "type": "metrics", + "name": "mongodb.metrics", + "namespace": "default" + }, + "ecs": { + "version": "1.5.0" } } \ No newline at end of file diff --git a/packages/mongodb/data_stream/replstatus/sample_event.json b/packages/mongodb/data_stream/replstatus/sample_event.json index 31204d57c64..04d5a09f750 100644 --- a/packages/mongodb/data_stream/replstatus/sample_event.json +++ b/packages/mongodb/data_stream/replstatus/sample_event.json @@ -1,53 +1,39 @@ { - "_index": ".ds-metrics-mongodb.replstatus-default-000001", - "_id": "3BT0AXMB-2lnjH4qREj0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.457Z", - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "error": { - "message": "error getting replication info: collection oplog.rs was not found" - }, - "dataset": { - "name": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "dataset": "mongodb.replstatus", - "module": "mongodb", - "duration": 1962467 - }, - "metricset": { - "name": "replstatus", - "period": 10000 - }, - "agent": { - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro", - "type": "metricbeat", - "version": "8.0.0" - } + "@timestamp": "2020-06-29T21:20:51.457Z", + "service": { + "address": "localhost:27017", + "type": "mongodb" }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.457Z" - ] + "error": { + "message": "error getting replication info: collection oplog.rs was not found" }, - "sort": [ - 1593465651457 - ] + "dataset": { + "name": "mongodb.replstatus", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "dataset": "mongodb.replstatus", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "mongodb.replstatus", + "module": "mongodb", + "duration": 1962467 + }, + "metricset": { + "name": "replstatus", + "period": 10000 + }, + "agent": { + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro", + "type": "metricbeat", + "version": "8.0.0" + } } \ No newline at end of file diff --git a/packages/mongodb/data_stream/status/sample_event.json b/packages/mongodb/data_stream/status/sample_event.json index c57df93cca6..a9ba62435bf 100644 --- a/packages/mongodb/data_stream/status/sample_event.json +++ b/packages/mongodb/data_stream/status/sample_event.json @@ -1,231 +1,214 @@ { - "_index": ".ds-metrics-mongodb.status-default-000001", - "_id": "ZxTzAXMB-2lnjH4qgUKh", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:01.455Z", - "dataset": { - "type": "metrics", - "name": "mongodb.status", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "mongodb.status", - "namespace": "default" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro", - "type": "metricbeat" - }, - "process": { - "name": "mongod" - }, - "event": { - "duration": 3581045, - "dataset": "mongodb.status", - "module": "mongodb" - }, - "mongodb": { - "status": { - "locks": { - "global": { - "acquire": { - "count": { - "w": 458, - "W": 4, - "r": 56961 - } - }, - "wait": {}, - "deadlock": {} + "@timestamp": "2020-06-29T21:20:01.455Z", + "dataset": { + "type": "metrics", + "name": "mongodb.status", + "namespace": "default" + }, + "stream": { + "type": "metrics", + "dataset": "mongodb.status", + "namespace": "default" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro", + "type": "metricbeat" + }, + "process": { + "name": "mongod" + }, + "event": { + "duration": 3581045, + "dataset": "mongodb.status", + "module": "mongodb" + }, + "mongodb": { + "status": { + "locks": { + "global": { + "acquire": { + "count": { + "w": 458, + "W": 4, + "r": 56961 + } }, - "database": { - "deadlock": {}, - "acquire": { - "count": { - "w": 453, - "W": 5, - "r": 5238 - } - }, - "wait": {} + "wait": {}, + "deadlock": {} + }, + "database": { + "deadlock": {}, + "acquire": { + "count": { + "w": 453, + "W": 5, + "r": 5238 + } }, - "collection": { - "wait": {}, - "deadlock": {}, - "acquire": { - "count": { - "W": 3, - "r": 8221, - "w": 450 - } + "wait": {} + }, + "collection": { + "wait": {}, + "deadlock": {}, + "acquire": { + "count": { + "W": 3, + "r": 8221, + "w": 450 } } + } + }, + "network": { + "in": { + "bytes": 687306 }, - "network": { - "in": { - "bytes": 687306 - }, - "out": { - "bytes": 32519464 - }, - "requests": 11607 - }, - "extra_info": { - "page_faults": 0, - "heap_usage": {} + "out": { + "bytes": 32519464 }, - "local_time": "2020-06-29T21:20:01.457Z", - "storage_engine": { - "name": "wiredTiger" + "requests": 11607 + }, + "extra_info": { + "page_faults": 0, + "heap_usage": {} + }, + "local_time": "2020-06-29T21:20:01.457Z", + "storage_engine": { + "name": "wiredTiger" + }, + "asserts": { + "user": 9, + "rollovers": 0, + "regular": 0, + "warning": 0, + "msg": 0 + }, + "global_lock": { + "total_time": { + "us": 26003338000 }, - "asserts": { - "user": 9, - "rollovers": 0, - "regular": 0, - "warning": 0, - "msg": 0 + "current_queue": { + "total": 0, + "readers": 0, + "writers": 0 }, - "global_lock": { - "total_time": { - "us": 26003338000 + "active_clients": { + "total": 1, + "readers": 1, + "writers": 0 + } + }, + "wired_tiger": { + "log": { + "syncs": 67, + "size": { + "bytes": 33554432 }, - "current_queue": { - "total": 0, - "readers": 0, - "writers": 0 + "write": { + "bytes": 46976 }, - "active_clients": { - "total": 1, - "readers": 1, - "writers": 0 - } - }, - "wired_tiger": { - "log": { - "syncs": 67, - "size": { - "bytes": 33554432 - }, - "write": { - "bytes": 46976 - }, - "max_file_size": { - "bytes": 104857600 - }, - "flushes": 152183, - "writes": 140, - "scans": 6 + "max_file_size": { + "bytes": 104857600 }, - "concurrent_transactions": { - "write": { - "out": 0, - "available": 128, - "total_tickets": 128 - }, - "read": { - "available": 128, - "total_tickets": 128, - "out": 0 - } + "flushes": 152183, + "writes": 140, + "scans": 6 + }, + "concurrent_transactions": { + "write": { + "out": 0, + "available": 128, + "total_tickets": 128 }, - "cache": { - "dirty": { - "bytes": 0 - }, - "pages": { - "evicted": 0, - "read": 14, - "write": 111 - }, - "maximum": { - "bytes": 16642998272 - }, - "used": { - "bytes": 89236 - } + "read": { + "available": 128, + "total_tickets": 128, + "out": 0 } }, - "memory": { - "mapped_with_journal": {}, - "bits": 64, - "resident": { - "mb": 44 + "cache": { + "dirty": { + "bytes": 0 + }, + "pages": { + "evicted": 0, + "read": 14, + "write": 111 }, - "virtual": { - "mb": 6971 + "maximum": { + "bytes": 16642998272 }, - "mapped": {} + "used": { + "bytes": 89236 + } + } + }, + "memory": { + "mapped_with_journal": {}, + "bits": 64, + "resident": { + "mb": 44 + }, + "virtual": { + "mb": 6971 + }, + "mapped": {} + }, + "connections": { + "total_created": 2266, + "current": 5, + "available": 3271 + }, + "ops": { + "counters": { + "delete": 3, + "getmore": 452, + "command": 11314, + "insert": 19, + "query": 94, + "update": 5 }, - "connections": { - "total_created": 2266, - "current": 5, - "available": 3271 + "replicated": { + "delete": 0, + "getmore": 0, + "command": 0, + "insert": 0, + "query": 0, + "update": 0 }, - "ops": { - "counters": { - "delete": 3, - "getmore": 452, - "command": 11314, - "insert": 19, - "query": 94, - "update": 5 + "latencies": { + "writes": { + "latency": 103455, + "count": 9 }, - "replicated": { - "delete": 0, - "getmore": 0, - "command": 0, - "insert": 0, - "query": 0, - "update": 0 + "commands": { + "latency": 2055949, + "count": 11138 }, - "latencies": { - "writes": { - "latency": 103455, - "count": 9 - }, - "commands": { - "latency": 2055949, - "count": 11138 - }, - "reads": { - "latency": 14259, - "count": 458 - } + "reads": { + "latency": 14259, + "count": 458 } - }, - "uptime": { - "ms": 26003340 } + }, + "uptime": { + "ms": 26003340 } - }, - "service": { - "version": "4.2.0", - "address": "localhost:27017", - "type": "mongodb" - }, - "metricset": { - "name": "status", - "period": 10000 - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:01.455Z" - ], - "mongodb.status.local_time": [ - "2020-06-29T21:20:01.457Z" - ] + "service": { + "version": "4.2.0", + "address": "localhost:27017", + "type": "mongodb" + }, + "metricset": { + "name": "status", + "period": 10000 }, - "sort": [ - 1593465601455 - ] + "ecs": { + "version": "1.5.0" + } } \ No newline at end of file diff --git a/packages/mongodb/docs/README.md b/packages/mongodb/docs/README.md index d7927190dca..6d2c16b987d 100644 --- a/packages/mongodb/docs/README.md +++ b/packages/mongodb/docs/README.md @@ -104,113 +104,102 @@ An example event for `collstats` looks as following: ```$json { - "_index": ".ds-metrics-mongodb.collstats-default-000001", - "_id": "6hT0AXMB-2lnjH4qREj1", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.459Z", - "metricset": { - "name": "collstats", - "period": 10000 - }, - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "stream": { - "dataset": "mongodb.collstats", - "namespace": "default", - "type": "metrics" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro" - }, - "event": { - "dataset": "mongodb.collstats", - "module": "mongodb", - "duration": 3378520 - }, - "mongodb": { - "collstats": { - "collection": "startup_log", - "commands": { - "count": 0, - "time": { - "us": 0 - } - }, - "db": "local", - "getmore": { - "count": 0, - "time": { - "us": 0 - } - }, - "insert": { - "count": 0, - "time": { - "us": 0 - } - }, - "lock": { - "read": { - "count": 74, - "time": { - "us": 443 - } - }, - "write": { - "count": 1, - "time": { - "us": 8 - } - } - }, - "name": "local.startup_log", - "queries": { - "count": 0, - "time": { - "us": 0 - } - }, - "remove": { - "count": 0, - "time": { - "us": 0 - } - }, - "total": { - "count": 75, + "@timestamp": "2020-06-29T21:20:51.459Z", + "metricset": { + "name": "collstats", + "period": 10000 + }, + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "stream": { + "dataset": "mongodb.collstats", + "namespace": "default", + "type": "metrics" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro" + }, + "event": { + "dataset": "mongodb.collstats", + "module": "mongodb", + "duration": 3378520 + }, + "mongodb": { + "collstats": { + "collection": "startup_log", + "commands": { + "count": 0, + "time": { + "us": 0 + } + }, + "db": "local", + "getmore": { + "count": 0, + "time": { + "us": 0 + } + }, + "insert": { + "count": 0, + "time": { + "us": 0 + } + }, + "lock": { + "read": { + "count": 74, "time": { - "us": 451 + "us": 443 } }, - "update": { - "count": 0, + "write": { + "count": 1, "time": { - "us": 0 + "us": 8 } } + }, + "name": "local.startup_log", + "queries": { + "count": 0, + "time": { + "us": 0 + } + }, + "remove": { + "count": 0, + "time": { + "us": 0 + } + }, + "total": { + "count": 75, + "time": { + "us": 451 + } + }, + "update": { + "count": 0, + "time": { + "us": 0 + } } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.collstats" - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.459Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "mongodb.collstats" + }, + "ecs": { + "version": "1.5.0" } } ``` @@ -294,73 +283,62 @@ An example event for `dbstats` looks as following: ```$json { - "_index": ".ds-metrics-mongodb.dbstats-default-000001", - "_id": "6hT0AXMB-2lnjH4qREj0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.459Z", - "metricset": { - "name": "dbstats", - "period": 10000 - }, - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "stream": { - "dataset": "mongodb.dbstats", - "namespace": "default", - "type": "metrics" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro" - }, - "event": { - "dataset": "mongodb.dbstats", - "module": "mongodb", - "duration": 3378520 - }, - "mongodb": { - "dbstats": { - "file_size": {}, - "index_size": { - "bytes": 20480 - }, - "ns_size_mb": {}, - "storage_size": { - "bytes": 20480 - }, - "num_extents": 0, - "collections": 1, - "objects": 1, - "db": "admin", - "data_size": { - "bytes": 59 - }, - "indexes": 1, - "avg_obj_size": { - "bytes": 59 - } + "@timestamp": "2020-06-29T21:20:51.459Z", + "metricset": { + "name": "dbstats", + "period": 10000 + }, + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "stream": { + "dataset": "mongodb.dbstats", + "namespace": "default", + "type": "metrics" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro" + }, + "event": { + "dataset": "mongodb.dbstats", + "module": "mongodb", + "duration": 3378520 + }, + "mongodb": { + "dbstats": { + "file_size": {}, + "index_size": { + "bytes": 20480 + }, + "ns_size_mb": {}, + "storage_size": { + "bytes": 20480 + }, + "num_extents": 0, + "collections": 1, + "objects": 1, + "db": "admin", + "data_size": { + "bytes": 59 + }, + "indexes": 1, + "avg_obj_size": { + "bytes": 59 } - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.dbstats" - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.459Z" - ] + "dataset": { + "namespace": "default", + "type": "metrics", + "name": "mongodb.dbstats" + }, + "ecs": { + "version": "1.5.0" } } ``` @@ -433,265 +411,254 @@ An example event for `metrics` looks as following: ```$json { - "_index": ".ds-metrics-mongodb.metrics-default-000001", - "_id": "6RT0AXMB-2lnjH4qREj0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.459Z", - "mongodb": { - "metrics": { - "replication": { - "network": { - "ops": 0, - "reders_created": 0, - "bytes": 0, - "getmores": { - "count": 0, - "time": { - "ms": 0 - } + "@timestamp": "2020-06-29T21:20:51.459Z", + "mongodb": { + "metrics": { + "replication": { + "network": { + "ops": 0, + "reders_created": 0, + "bytes": 0, + "getmores": { + "count": 0, + "time": { + "ms": 0 } - }, - "executor": { - "shutting_down": false, - "network_interface": "DEPRECATED: getDiagnosticString is deprecated in NetworkInterfaceTL", - "queues": { - "in_progress": { - "network": 0 - }, - "sleepers": 0 - }, - "unsignaled_events": 0 - }, - "apply": { - "attempts_to_become_secondary": 0, - "batches": { - "count": 0, - "time": { - "ms": 0 - } - }, - "ops": 0 - }, - "buffer": { - "max_size": { - "bytes": 0 - }, - "size": { - "bytes": 0 - }, - "count": 0 - }, - "initial_sync": { - "completed": 0, - "failed_attempts": 0, - "failures": 0 } }, - "ttl": { - "passes": { - "count": 433 + "executor": { + "shutting_down": false, + "network_interface": "DEPRECATED: getDiagnosticString is deprecated in NetworkInterfaceTL", + "queues": { + "in_progress": { + "network": 0 + }, + "sleepers": 0 }, - "deleted_documents": { - "count": 3 - } + "unsignaled_events": 0 }, - "commands": { - "replset_heartbeat": { - "failed": 0, - "total": 0 - }, - "connection_pool_stats": { - "failed": 0, - "total": 0 - }, - "host_info": { - "failed": 0, - "total": 0 - }, - "aggregate": { - "failed": 0, - "total": 0 - }, - "replset_update_position": { - "total": 0, - "failed": 0 - }, - "last_collections": { - "failed": 0, - "total": 458 - }, - "list_databased": { - "total": 466, - "failed": 0 - }, - "whatsmyuri": { - "total": 2, - "failed": 0 - }, - "profile": { - "failed": 0, - "total": 0 - }, - "insert": { - "failed": 0, - "total": 7 - }, - "count": { - "failed": 0, - "total": 0 - }, - "is_master": { - "failed": 0, - "total": 2332 - }, - "distinct": { - "failed": 0, - "total": 0 - }, - "replset_get_status": { - "failed": 2, - "total": 2 - }, - "find": { - "failed": 0, - "total": 94 - }, - "replset_get_rbid": { - "failed": 0, - "total": 0 - }, - "get_parameter": { - "failed": 0, - "total": 0 - }, - "coll_stats": { - "failed": 0, - "total": 0 - }, - "build_info": { - "total": 6, - "failed": 0 - }, - "last_commands": { - "failed": 0, - "total": 0 - }, - "update": { - "failed": 0, - "total": 5 - }, - "is_self": { - "failed": 0, - "total": 0 - }, - "db_stats": { - "failed": 0, - "total": 2044 - }, - "get_cmd_line_opts": { - "failed": 0, - "total": 2 - }, - "ping": { - "total": 2290, - "failed": 0 - }, - "server_status": { - "total": 916, - "failed": 0 - }, - "get_last_error": { - "failed": 0, - "total": 0 + "apply": { + "attempts_to_become_secondary": 0, + "batches": { + "count": 0, + "time": { + "ms": 0 + } }, - "get_more": { - "failed": 0, - "total": 0 + "ops": 0 + }, + "buffer": { + "max_size": { + "bytes": 0 }, - "get_log": { - "failed": 0, - "total": 2 + "size": { + "bytes": 0 }, - "list_indexes": { - "failed": 0, - "total": 174 - } + "count": 0 }, - "cursor": { - "timed_out": 0, - "open": { - "pinned": 0, - "total": 0, - "no_timeout": 0 - } + "initial_sync": { + "completed": 0, + "failed_attempts": 0, + "failures": 0 + } + }, + "ttl": { + "passes": { + "count": 433 + }, + "deleted_documents": { + "count": 3 + } + }, + "commands": { + "replset_heartbeat": { + "failed": 0, + "total": 0 + }, + "connection_pool_stats": { + "failed": 0, + "total": 0 + }, + "host_info": { + "failed": 0, + "total": 0 + }, + "aggregate": { + "failed": 0, + "total": 0 + }, + "replset_update_position": { + "total": 0, + "failed": 0 + }, + "last_collections": { + "failed": 0, + "total": 458 + }, + "list_databased": { + "total": 466, + "failed": 0 + }, + "whatsmyuri": { + "total": 2, + "failed": 0 + }, + "profile": { + "failed": 0, + "total": 0 + }, + "insert": { + "failed": 0, + "total": 7 + }, + "count": { + "failed": 0, + "total": 0 + }, + "is_master": { + "failed": 0, + "total": 2332 + }, + "distinct": { + "failed": 0, + "total": 0 + }, + "replset_get_status": { + "failed": 2, + "total": 2 + }, + "find": { + "failed": 0, + "total": 94 + }, + "replset_get_rbid": { + "failed": 0, + "total": 0 + }, + "get_parameter": { + "failed": 0, + "total": 0 + }, + "coll_stats": { + "failed": 0, + "total": 0 + }, + "build_info": { + "total": 6, + "failed": 0 + }, + "last_commands": { + "failed": 0, + "total": 0 + }, + "update": { + "failed": 0, + "total": 5 + }, + "is_self": { + "failed": 0, + "total": 0 + }, + "db_stats": { + "failed": 0, + "total": 2044 + }, + "get_cmd_line_opts": { + "failed": 0, + "total": 2 + }, + "ping": { + "total": 2290, + "failed": 0 + }, + "server_status": { + "total": 916, + "failed": 0 }, "get_last_error": { - "write_wait": { - "ms": 0, - "count": 0 - }, - "write_timeouts": 0 + "failed": 0, + "total": 0 }, - "operation": { - "write_conflicts": 0, - "scan_and_order": 0 + "get_more": { + "failed": 0, + "total": 0 }, - "document": { - "deleted": 15, - "inserted": 19, - "returned": 465, - "updated": 2 + "get_log": { + "failed": 0, + "total": 2 }, - "query_executor": { - "scanned_indexes": { - "count": 2 - }, - "scanned_documents": { - "count": 24 - } + "list_indexes": { + "failed": 0, + "total": 174 + } + }, + "cursor": { + "timed_out": 0, + "open": { + "pinned": 0, + "total": 0, + "no_timeout": 0 + } + }, + "get_last_error": { + "write_wait": { + "ms": 0, + "count": 0 + }, + "write_timeouts": 0 + }, + "operation": { + "write_conflicts": 0, + "scan_and_order": 0 + }, + "document": { + "deleted": 15, + "inserted": 19, + "returned": 465, + "updated": 2 + }, + "query_executor": { + "scanned_indexes": { + "count": 2 + }, + "scanned_documents": { + "count": 24 } } - }, - "metricset": { - "period": 10000, - "name": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "mongodb.metrics", - "namespace": "default" - }, - "agent": { - "name": "KaiyanMacBookPro", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a" - }, - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "event": { - "dataset": "mongodb.metrics", - "module": "mongodb", - "duration": 3039885 - }, - "dataset": { - "type": "metrics", - "name": "mongodb.metrics", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.459Z" - ] + "metricset": { + "period": 10000, + "name": "metrics" + }, + "stream": { + "type": "metrics", + "dataset": "mongodb.metrics", + "namespace": "default" + }, + "agent": { + "name": "KaiyanMacBookPro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a" + }, + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "event": { + "dataset": "mongodb.metrics", + "module": "mongodb", + "duration": 3039885 + }, + "dataset": { + "type": "metrics", + "name": "mongodb.metrics", + "namespace": "default" + }, + "ecs": { + "version": "1.5.0" } } ``` @@ -869,57 +836,43 @@ An example event for `replstatus` looks as following: ```$json { - "_index": ".ds-metrics-mongodb.replstatus-default-000001", - "_id": "3BT0AXMB-2lnjH4qREj0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:51.457Z", - "service": { - "address": "localhost:27017", - "type": "mongodb" - }, - "error": { - "message": "error getting replication info: collection oplog.rs was not found" - }, - "dataset": { - "name": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "dataset": "mongodb.replstatus", - "module": "mongodb", - "duration": 1962467 - }, - "metricset": { - "name": "replstatus", - "period": 10000 - }, - "agent": { - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro", - "type": "metricbeat", - "version": "8.0.0" - } + "@timestamp": "2020-06-29T21:20:51.457Z", + "service": { + "address": "localhost:27017", + "type": "mongodb" + }, + "error": { + "message": "error getting replication info: collection oplog.rs was not found" + }, + "dataset": { + "name": "mongodb.replstatus", + "namespace": "default", + "type": "metrics" + }, + "stream": { + "dataset": "mongodb.replstatus", + "namespace": "default", + "type": "metrics" }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:51.457Z" - ] + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1593465651457 - ] + "event": { + "dataset": "mongodb.replstatus", + "module": "mongodb", + "duration": 1962467 + }, + "metricset": { + "name": "replstatus", + "period": 10000 + }, + "agent": { + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro", + "type": "metricbeat", + "version": "8.0.0" + } } ``` @@ -1009,235 +962,218 @@ An example event for `status` looks as following: ```$json { - "_index": ".ds-metrics-mongodb.status-default-000001", - "_id": "ZxTzAXMB-2lnjH4qgUKh", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T21:20:01.455Z", - "dataset": { - "type": "metrics", - "name": "mongodb.status", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "mongodb.status", - "namespace": "default" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", - "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", - "name": "KaiyanMacBookPro", - "type": "metricbeat" - }, - "process": { - "name": "mongod" - }, - "event": { - "duration": 3581045, - "dataset": "mongodb.status", - "module": "mongodb" - }, - "mongodb": { - "status": { - "locks": { - "global": { - "acquire": { - "count": { - "w": 458, - "W": 4, - "r": 56961 - } - }, - "wait": {}, - "deadlock": {} + "@timestamp": "2020-06-29T21:20:01.455Z", + "dataset": { + "type": "metrics", + "name": "mongodb.status", + "namespace": "default" + }, + "stream": { + "type": "metrics", + "dataset": "mongodb.status", + "namespace": "default" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", + "id": "2281e192-85d5-4d68-b90a-36a31df7b29a", + "name": "KaiyanMacBookPro", + "type": "metricbeat" + }, + "process": { + "name": "mongod" + }, + "event": { + "duration": 3581045, + "dataset": "mongodb.status", + "module": "mongodb" + }, + "mongodb": { + "status": { + "locks": { + "global": { + "acquire": { + "count": { + "w": 458, + "W": 4, + "r": 56961 + } }, - "database": { - "deadlock": {}, - "acquire": { - "count": { - "w": 453, - "W": 5, - "r": 5238 - } - }, - "wait": {} + "wait": {}, + "deadlock": {} + }, + "database": { + "deadlock": {}, + "acquire": { + "count": { + "w": 453, + "W": 5, + "r": 5238 + } }, - "collection": { - "wait": {}, - "deadlock": {}, - "acquire": { - "count": { - "W": 3, - "r": 8221, - "w": 450 - } + "wait": {} + }, + "collection": { + "wait": {}, + "deadlock": {}, + "acquire": { + "count": { + "W": 3, + "r": 8221, + "w": 450 } } + } + }, + "network": { + "in": { + "bytes": 687306 }, - "network": { - "in": { - "bytes": 687306 - }, - "out": { - "bytes": 32519464 - }, - "requests": 11607 - }, - "extra_info": { - "page_faults": 0, - "heap_usage": {} + "out": { + "bytes": 32519464 }, - "local_time": "2020-06-29T21:20:01.457Z", - "storage_engine": { - "name": "wiredTiger" + "requests": 11607 + }, + "extra_info": { + "page_faults": 0, + "heap_usage": {} + }, + "local_time": "2020-06-29T21:20:01.457Z", + "storage_engine": { + "name": "wiredTiger" + }, + "asserts": { + "user": 9, + "rollovers": 0, + "regular": 0, + "warning": 0, + "msg": 0 + }, + "global_lock": { + "total_time": { + "us": 26003338000 }, - "asserts": { - "user": 9, - "rollovers": 0, - "regular": 0, - "warning": 0, - "msg": 0 + "current_queue": { + "total": 0, + "readers": 0, + "writers": 0 }, - "global_lock": { - "total_time": { - "us": 26003338000 + "active_clients": { + "total": 1, + "readers": 1, + "writers": 0 + } + }, + "wired_tiger": { + "log": { + "syncs": 67, + "size": { + "bytes": 33554432 }, - "current_queue": { - "total": 0, - "readers": 0, - "writers": 0 + "write": { + "bytes": 46976 }, - "active_clients": { - "total": 1, - "readers": 1, - "writers": 0 - } - }, - "wired_tiger": { - "log": { - "syncs": 67, - "size": { - "bytes": 33554432 - }, - "write": { - "bytes": 46976 - }, - "max_file_size": { - "bytes": 104857600 - }, - "flushes": 152183, - "writes": 140, - "scans": 6 + "max_file_size": { + "bytes": 104857600 }, - "concurrent_transactions": { - "write": { - "out": 0, - "available": 128, - "total_tickets": 128 - }, - "read": { - "available": 128, - "total_tickets": 128, - "out": 0 - } + "flushes": 152183, + "writes": 140, + "scans": 6 + }, + "concurrent_transactions": { + "write": { + "out": 0, + "available": 128, + "total_tickets": 128 }, - "cache": { - "dirty": { - "bytes": 0 - }, - "pages": { - "evicted": 0, - "read": 14, - "write": 111 - }, - "maximum": { - "bytes": 16642998272 - }, - "used": { - "bytes": 89236 - } + "read": { + "available": 128, + "total_tickets": 128, + "out": 0 } }, - "memory": { - "mapped_with_journal": {}, - "bits": 64, - "resident": { - "mb": 44 + "cache": { + "dirty": { + "bytes": 0 }, - "virtual": { - "mb": 6971 + "pages": { + "evicted": 0, + "read": 14, + "write": 111 }, - "mapped": {} + "maximum": { + "bytes": 16642998272 + }, + "used": { + "bytes": 89236 + } + } + }, + "memory": { + "mapped_with_journal": {}, + "bits": 64, + "resident": { + "mb": 44 }, - "connections": { - "total_created": 2266, - "current": 5, - "available": 3271 + "virtual": { + "mb": 6971 }, - "ops": { - "counters": { - "delete": 3, - "getmore": 452, - "command": 11314, - "insert": 19, - "query": 94, - "update": 5 + "mapped": {} + }, + "connections": { + "total_created": 2266, + "current": 5, + "available": 3271 + }, + "ops": { + "counters": { + "delete": 3, + "getmore": 452, + "command": 11314, + "insert": 19, + "query": 94, + "update": 5 + }, + "replicated": { + "delete": 0, + "getmore": 0, + "command": 0, + "insert": 0, + "query": 0, + "update": 0 + }, + "latencies": { + "writes": { + "latency": 103455, + "count": 9 }, - "replicated": { - "delete": 0, - "getmore": 0, - "command": 0, - "insert": 0, - "query": 0, - "update": 0 + "commands": { + "latency": 2055949, + "count": 11138 }, - "latencies": { - "writes": { - "latency": 103455, - "count": 9 - }, - "commands": { - "latency": 2055949, - "count": 11138 - }, - "reads": { - "latency": 14259, - "count": 458 - } + "reads": { + "latency": 14259, + "count": 458 } - }, - "uptime": { - "ms": 26003340 } + }, + "uptime": { + "ms": 26003340 } - }, - "service": { - "version": "4.2.0", - "address": "localhost:27017", - "type": "mongodb" - }, - "metricset": { - "name": "status", - "period": 10000 - }, - "ecs": { - "version": "1.5.0" } }, - "fields": { - "@timestamp": [ - "2020-06-29T21:20:01.455Z" - ], - "mongodb.status.local_time": [ - "2020-06-29T21:20:01.457Z" - ] + "service": { + "version": "4.2.0", + "address": "localhost:27017", + "type": "mongodb" }, - "sort": [ - 1593465601455 - ] + "metricset": { + "name": "status", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + } } ``` diff --git a/packages/prometheus/data_stream/collector/sample_event.json b/packages/prometheus/data_stream/collector/sample_event.json index 8b5ec253983..152380e0419 100644 --- a/packages/prometheus/data_stream/collector/sample_event.json +++ b/packages/prometheus/data_stream/collector/sample_event.json @@ -1,56 +1,37 @@ { - "_index": ".ds-metrics-prometheus.collector-default-000001", - "_id": "xhalI3MBp-HYgBGo7zEW", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T10:22:23.034Z", - "agent": {}, - "event": { - "dataset": "prometheus.collector", - "module": "prometheus", - "duration": 13290705 - }, - "metricset": { - "name": "collector", - "period": 10000 - }, - "service": { - "address": "localhost:9090", - "type": "prometheus" - }, - "prometheus": { - "metrics": { - "prometheus_wal_watcher_records_read_total": 74 - }, - "labels": { - "job": "prometheus", - "consumer": "ee9cb2", - "type": "series", - "instance": "localhost:9090" - } - }, - "dataset": { - "type": "metrics", - "name": "prometheus.collector", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-07-06T10:22:23.034Z", + "agent": {}, + "event": { + "dataset": "prometheus.collector", + "module": "prometheus", + "duration": 13290705 + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "service": { + "address": "localhost:9090", + "type": "prometheus" + }, + "prometheus": { + "metrics": { + "prometheus_wal_watcher_records_read_total": 74 }, - "host": {} + "labels": { + "job": "prometheus", + "consumer": "ee9cb2", + "type": "series", + "instance": "localhost:9090" + } }, - "fields": { - "@timestamp": [ - "2020-07-06T10:22:23.034Z" - ] + "dataset": { + "type": "metrics", + "name": "prometheus.collector", + "namespace": "default" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@prometheus.collector@/kibana-highlighted-field@" - ] + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1594030943034 - ] + "host": {} } \ No newline at end of file diff --git a/packages/prometheus/data_stream/query/sample_event.json b/packages/prometheus/data_stream/query/sample_event.json index f173bcb8f7f..7a880a964d8 100644 --- a/packages/prometheus/data_stream/query/sample_event.json +++ b/packages/prometheus/data_stream/query/sample_event.json @@ -1,62 +1,43 @@ { - "_index": ".ds-metrics-prometheus.query-default-000001", - "_id": "IlG5AHMBeyDc0b9rYc28", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T15:36:54.000Z", - "host": {}, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "98420e91-ee6d-4883-8ad3-02fa8d47f5c1", - "id": "9fc3e975-6789-4738-a11a-ba7108b0a92c", - "name": "minikube" - }, - "event": { - "module": "prometheus", - "duration": 2123733, - "dataset": "prometheus.query" - }, - "metricset": { - "name": "query", - "period": 10000 - }, - "dataset": { - "type": "metrics", - "name": "prometheus.query", - "namespace": "default" - }, - "stream": { - "dataset": "prometheus.query", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "service": { - "address": "localhost:9090", - "type": "prometheus" - }, - "prometheus": { - "labels": {}, - "query": { - "prometheus_http_requests_total_rate": 0.3818181818181818 - } - } + "@timestamp": "2020-06-29T15:36:54.000Z", + "host": {}, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "98420e91-ee6d-4883-8ad3-02fa8d47f5c1", + "id": "9fc3e975-6789-4738-a11a-ba7108b0a92c", + "name": "minikube" + }, + "event": { + "module": "prometheus", + "duration": 2123733, + "dataset": "prometheus.query" + }, + "metricset": { + "name": "query", + "period": 10000 }, - "fields": { - "@timestamp": [ - "2020-06-29T15:36:54.000Z" - ] + "dataset": { + "type": "metrics", + "name": "prometheus.query", + "namespace": "default" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@prometheus.query@/kibana-highlighted-field@" - ] + "stream": { + "dataset": "prometheus.query", + "namespace": "default", + "type": "metrics" }, - "sort": [ - 1593445014000 - ] + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "localhost:9090", + "type": "prometheus" + }, + "prometheus": { + "labels": {}, + "query": { + "prometheus_http_requests_total_rate": 0.3818181818181818 + } + } } \ No newline at end of file diff --git a/packages/prometheus/data_stream/remote_write/sample_event.json b/packages/prometheus/data_stream/remote_write/sample_event.json index 4c1cb060c02..3458d4ba3bd 100644 --- a/packages/prometheus/data_stream/remote_write/sample_event.json +++ b/packages/prometheus/data_stream/remote_write/sample_event.json @@ -1,54 +1,40 @@ { - "_index": ".ds-metrics-prometheus.remote_write-default-000001", - "_id": "dJf5AHMBA2PIMpu1O4DQ", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T16:46:40.018Z", - "ecs": { - "version": "1.5.0" - }, - "host": {}, - "agent": { - "version": "8.0.0", - "ephemeral_id": "cb348102-0121-4c5b-8fcd-10ea27d25f77", - "id": "3bdc7670-9ced-4c70-bba9-00d7e183ae4b", - "name": "Christoss-MBP", - "type": "metricbeat" - }, - "metricset": { - "name": "remote_write" - }, - "prometheus": { - "metrics": { - "container_fs_reads_bytes_total": 1196032, - "container_fs_reads_total": 27 - }, - "labels": { - "instance": "cadvisor:8080", - "job": "cadvisor", - "id": "/systemreserved/acpid" - } - }, - "service": { - "type": "prometheus" - }, - "event": { - "dataset": "prometheus.remote_write", - "module": "prometheus" + "@timestamp": "2020-06-29T16:46:40.018Z", + "ecs": { + "version": "1.5.0" + }, + "host": {}, + "agent": { + "version": "8.0.0", + "ephemeral_id": "cb348102-0121-4c5b-8fcd-10ea27d25f77", + "id": "3bdc7670-9ced-4c70-bba9-00d7e183ae4b", + "name": "Christoss-MBP", + "type": "metricbeat" + }, + "metricset": { + "name": "remote_write" + }, + "prometheus": { + "metrics": { + "container_fs_reads_bytes_total": 1196032, + "container_fs_reads_total": 27 }, - "dataset": { - "type": "metrics", - "name": "prometheus.remote_write", - "namespace": "default" + "labels": { + "instance": "cadvisor:8080", + "job": "cadvisor", + "id": "/systemreserved/acpid" } }, - "fields": { - "@timestamp": [ - "2020-06-29T16:46:40.018Z" - ] + "service": { + "type": "prometheus" + }, + "event": { + "dataset": "prometheus.remote_write", + "module": "prometheus" }, - "sort": [ - 1593449200018 - ] + "dataset": { + "type": "metrics", + "name": "prometheus.remote_write", + "namespace": "default" + } } \ No newline at end of file diff --git a/packages/prometheus/docs/README.md b/packages/prometheus/docs/README.md index 292c426bffd..e8756261b10 100644 --- a/packages/prometheus/docs/README.md +++ b/packages/prometheus/docs/README.md @@ -130,60 +130,41 @@ An example event for `collector` looks as following: ```$json { - "_index": ".ds-metrics-prometheus.collector-default-000001", - "_id": "xhalI3MBp-HYgBGo7zEW", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T10:22:23.034Z", - "agent": {}, - "event": { - "dataset": "prometheus.collector", - "module": "prometheus", - "duration": 13290705 - }, - "metricset": { - "name": "collector", - "period": 10000 - }, - "service": { - "address": "localhost:9090", - "type": "prometheus" - }, - "prometheus": { - "metrics": { - "prometheus_wal_watcher_records_read_total": 74 - }, - "labels": { - "job": "prometheus", - "consumer": "ee9cb2", - "type": "series", - "instance": "localhost:9090" - } - }, - "dataset": { - "type": "metrics", - "name": "prometheus.collector", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-07-06T10:22:23.034Z", + "agent": {}, + "event": { + "dataset": "prometheus.collector", + "module": "prometheus", + "duration": 13290705 + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "service": { + "address": "localhost:9090", + "type": "prometheus" + }, + "prometheus": { + "metrics": { + "prometheus_wal_watcher_records_read_total": 74 }, - "host": {} + "labels": { + "job": "prometheus", + "consumer": "ee9cb2", + "type": "series", + "instance": "localhost:9090" + } }, - "fields": { - "@timestamp": [ - "2020-07-06T10:22:23.034Z" - ] + "dataset": { + "type": "metrics", + "name": "prometheus.collector", + "namespace": "default" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@prometheus.collector@/kibana-highlighted-field@" - ] + "ecs": { + "version": "1.5.0" }, - "sort": [ - 1594030943034 - ] + "host": {} } ``` @@ -297,58 +278,44 @@ An example event for `remote_write` looks as following: ```$json { - "_index": ".ds-metrics-prometheus.remote_write-default-000001", - "_id": "dJf5AHMBA2PIMpu1O4DQ", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T16:46:40.018Z", - "ecs": { - "version": "1.5.0" - }, - "host": {}, - "agent": { - "version": "8.0.0", - "ephemeral_id": "cb348102-0121-4c5b-8fcd-10ea27d25f77", - "id": "3bdc7670-9ced-4c70-bba9-00d7e183ae4b", - "name": "Christoss-MBP", - "type": "metricbeat" - }, - "metricset": { - "name": "remote_write" - }, - "prometheus": { - "metrics": { - "container_fs_reads_bytes_total": 1196032, - "container_fs_reads_total": 27 - }, - "labels": { - "instance": "cadvisor:8080", - "job": "cadvisor", - "id": "/systemreserved/acpid" - } - }, - "service": { - "type": "prometheus" - }, - "event": { - "dataset": "prometheus.remote_write", - "module": "prometheus" + "@timestamp": "2020-06-29T16:46:40.018Z", + "ecs": { + "version": "1.5.0" + }, + "host": {}, + "agent": { + "version": "8.0.0", + "ephemeral_id": "cb348102-0121-4c5b-8fcd-10ea27d25f77", + "id": "3bdc7670-9ced-4c70-bba9-00d7e183ae4b", + "name": "Christoss-MBP", + "type": "metricbeat" + }, + "metricset": { + "name": "remote_write" + }, + "prometheus": { + "metrics": { + "container_fs_reads_bytes_total": 1196032, + "container_fs_reads_total": 27 }, - "dataset": { - "type": "metrics", - "name": "prometheus.remote_write", - "namespace": "default" + "labels": { + "instance": "cadvisor:8080", + "job": "cadvisor", + "id": "/systemreserved/acpid" } }, - "fields": { - "@timestamp": [ - "2020-06-29T16:46:40.018Z" - ] + "service": { + "type": "prometheus" }, - "sort": [ - 1593449200018 - ] + "event": { + "dataset": "prometheus.remote_write", + "module": "prometheus" + }, + "dataset": { + "type": "metrics", + "name": "prometheus.remote_write", + "namespace": "default" + } } ``` @@ -521,66 +488,47 @@ An example event for `query` looks as following: ```$json { - "_index": ".ds-metrics-prometheus.query-default-000001", - "_id": "IlG5AHMBeyDc0b9rYc28", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-29T15:36:54.000Z", - "host": {}, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "98420e91-ee6d-4883-8ad3-02fa8d47f5c1", - "id": "9fc3e975-6789-4738-a11a-ba7108b0a92c", - "name": "minikube" - }, - "event": { - "module": "prometheus", - "duration": 2123733, - "dataset": "prometheus.query" - }, - "metricset": { - "name": "query", - "period": 10000 - }, - "dataset": { - "type": "metrics", - "name": "prometheus.query", - "namespace": "default" - }, - "stream": { - "dataset": "prometheus.query", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "service": { - "address": "localhost:9090", - "type": "prometheus" - }, - "prometheus": { - "labels": {}, - "query": { - "prometheus_http_requests_total_rate": 0.3818181818181818 - } - } + "@timestamp": "2020-06-29T15:36:54.000Z", + "host": {}, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "98420e91-ee6d-4883-8ad3-02fa8d47f5c1", + "id": "9fc3e975-6789-4738-a11a-ba7108b0a92c", + "name": "minikube" + }, + "event": { + "module": "prometheus", + "duration": 2123733, + "dataset": "prometheus.query" }, - "fields": { - "@timestamp": [ - "2020-06-29T15:36:54.000Z" - ] + "metricset": { + "name": "query", + "period": 10000 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@prometheus.query@/kibana-highlighted-field@" - ] + "dataset": { + "type": "metrics", + "name": "prometheus.query", + "namespace": "default" }, - "sort": [ - 1593445014000 - ] + "stream": { + "dataset": "prometheus.query", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "localhost:9090", + "type": "prometheus" + }, + "prometheus": { + "labels": {}, + "query": { + "prometheus_http_requests_total_rate": 0.3818181818181818 + } + } } ``` diff --git a/packages/zookeeper/data_stream/connection/sample_event.json b/packages/zookeeper/data_stream/connection/sample_event.json index 79bb7095348..de73535b679 100644 --- a/packages/zookeeper/data_stream/connection/sample_event.json +++ b/packages/zookeeper/data_stream/connection/sample_event.json @@ -1,52 +1,41 @@ { - "_index": "metricbeat-8.0.0-2020.07.06-000001", - "_id": "5KTmJHMB--B0K1AVImYo", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T16:12:07.612Z", - "host": { - "name": "zookeeper-01" - }, - "metricset": { - "name": "connection", - "period": 10000 - }, - "service": { - "address": "localhost:2181", - "type": "zookeeper" - }, - "zookeeper": { - "connection": { - "received": 1, - "sent": 0, - "interest_ops": 0, - "queued": 0 - } - }, - "client": { - "ip": "172.28.0.1", - "port": 44338 - }, - "event": { - "dataset": "zookeeper.connection", - "module": "zookeeper", - "duration": 3093417 - }, - "agent": { - "name": "zookeeper-01", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", - "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-07-06T16:12:07.612Z", + "host": { + "name": "zookeeper-01" + }, + "metricset": { + "name": "connection", + "period": 10000 + }, + "service": { + "address": "localhost:2181", + "type": "zookeeper" + }, + "zookeeper": { + "connection": { + "received": 1, + "sent": 0, + "interest_ops": 0, + "queued": 0 } }, - "fields": { - "@timestamp": [ - "2020-07-06T16:12:07.612Z" - ] + "client": { + "ip": "172.28.0.1", + "port": 44338 + }, + "event": { + "dataset": "zookeeper.connection", + "module": "zookeeper", + "duration": 3093417 + }, + "agent": { + "name": "zookeeper-01", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", + "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" + }, + "ecs": { + "version": "1.5.0" } } \ No newline at end of file diff --git a/packages/zookeeper/data_stream/mntr/sample_event.json b/packages/zookeeper/data_stream/mntr/sample_event.json index 2500a9850c8..b134d83a112 100644 --- a/packages/zookeeper/data_stream/mntr/sample_event.json +++ b/packages/zookeeper/data_stream/mntr/sample_event.json @@ -1,63 +1,52 @@ { - "_index": "metricbeat-8.0.0-2020.07.06-000001", - "_id": "5aTmJHMB--B0K1AVImYo", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T16:12:08.494Z", - "zookeeper": { - "mntr": { - "open_file_descriptor_count": 49, - "watch_count": 0, - "server_state": "standalone", - "max_file_descriptor_count": 1048576, - "znode_count": 5, - "outstanding_requests": 0, - "ephemerals_count": 0, - "packets": { - "received": 152, - "sent": 151 - }, - "num_alive_connections": 1, - "approximate_data_size": 44, - "latency": { - "max": 0, - "avg": 0, - "min": 0 - } + "@timestamp": "2020-07-06T16:12:08.494Z", + "zookeeper": { + "mntr": { + "open_file_descriptor_count": 49, + "watch_count": 0, + "server_state": "standalone", + "max_file_descriptor_count": 1048576, + "znode_count": 5, + "outstanding_requests": 0, + "ephemerals_count": 0, + "packets": { + "received": 152, + "sent": 151 + }, + "num_alive_connections": 1, + "approximate_data_size": 44, + "latency": { + "max": 0, + "avg": 0, + "min": 0 } - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "zookeeper-01" - }, - "agent": { - "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", - "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651", - "name": "zookeeper-01", - "type": "metricbeat", - "version": "8.0.0" - }, - "service": { - "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653, built on 05/03/2019 12:07 GMT", - "address": "localhost:2181", - "type": "zookeeper" - }, - "event": { - "duration": 15795652, - "dataset": "zookeeper.mntr", - "module": "zookeeper" - }, - "metricset": { - "name": "mntr", - "period": 10000 } }, - "fields": { - "@timestamp": [ - "2020-07-06T16:12:08.494Z" - ] + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "zookeeper-01" + }, + "agent": { + "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", + "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651", + "name": "zookeeper-01", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653, built on 05/03/2019 12:07 GMT", + "address": "localhost:2181", + "type": "zookeeper" + }, + "event": { + "duration": 15795652, + "dataset": "zookeeper.mntr", + "module": "zookeeper" + }, + "metricset": { + "name": "mntr", + "period": 10000 } } \ No newline at end of file diff --git a/packages/zookeeper/data_stream/server/sample_event.json b/packages/zookeeper/data_stream/server/sample_event.json index efab518ff9a..e0e6b73e65a 100644 --- a/packages/zookeeper/data_stream/server/sample_event.json +++ b/packages/zookeeper/data_stream/server/sample_event.json @@ -1,63 +1,49 @@ { - "_index": "metricbeat-8.0.0-2020.07.06-000001", - "_id": "QKTmJHMB--B0K1AVNGfq", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T16:12:12.409Z", - "event": { - "module": "zookeeper", - "duration": 3001938, - "dataset": "zookeeper.server" - }, - "metricset": { - "name": "server", - "period": 10000 - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "zookeeper-01" - }, - "agent": { - "name": "zookeeper-01", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", - "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" - }, - "zookeeper": { - "server": { - "zxid": "0x0", - "count": 0, - "version_date": "2019-05-03T12:07:00Z", - "received": 156, - "mode": "standalone", - "latency": { - "avg": 0, - "max": 0, - "min": 0 - }, - "sent": 155, - "epoch": 0, - "node_count": 5, - "connections": 1, - "outstanding": 0 - } - }, - "service": { - "address": "localhost:2181", - "type": "zookeeper", - "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653" + "@timestamp": "2020-07-06T16:12:12.409Z", + "event": { + "module": "zookeeper", + "duration": 3001938, + "dataset": "zookeeper.server" + }, + "metricset": { + "name": "server", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "zookeeper-01" + }, + "agent": { + "name": "zookeeper-01", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", + "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" + }, + "zookeeper": { + "server": { + "zxid": "0x0", + "count": 0, + "version_date": "2019-05-03T12:07:00Z", + "received": 156, + "mode": "standalone", + "latency": { + "avg": 0, + "max": 0, + "min": 0 + }, + "sent": 155, + "epoch": 0, + "node_count": 5, + "connections": 1, + "outstanding": 0 } }, - "fields": { - "zookeeper.server.version_date": [ - "2019-05-03T12:07:00.000Z" - ], - "@timestamp": [ - "2020-07-06T16:12:12.409Z" - ] + "service": { + "address": "localhost:2181", + "type": "zookeeper", + "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653" } } \ No newline at end of file diff --git a/packages/zookeeper/docs/README.md b/packages/zookeeper/docs/README.md index 154c9e5b252..e8766e87f94 100644 --- a/packages/zookeeper/docs/README.md +++ b/packages/zookeeper/docs/README.md @@ -16,55 +16,44 @@ An example event for `connection` looks as following: ```$json { - "_index": "metricbeat-8.0.0-2020.07.06-000001", - "_id": "5KTmJHMB--B0K1AVImYo", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T16:12:07.612Z", - "host": { - "name": "zookeeper-01" - }, - "metricset": { - "name": "connection", - "period": 10000 - }, - "service": { - "address": "localhost:2181", - "type": "zookeeper" - }, - "zookeeper": { - "connection": { - "received": 1, - "sent": 0, - "interest_ops": 0, - "queued": 0 - } - }, - "client": { - "ip": "172.28.0.1", - "port": 44338 - }, - "event": { - "dataset": "zookeeper.connection", - "module": "zookeeper", - "duration": 3093417 - }, - "agent": { - "name": "zookeeper-01", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", - "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-07-06T16:12:07.612Z", + "host": { + "name": "zookeeper-01" + }, + "metricset": { + "name": "connection", + "period": 10000 + }, + "service": { + "address": "localhost:2181", + "type": "zookeeper" + }, + "zookeeper": { + "connection": { + "received": 1, + "sent": 0, + "interest_ops": 0, + "queued": 0 } }, - "fields": { - "@timestamp": [ - "2020-07-06T16:12:07.612Z" - ] + "client": { + "ip": "172.28.0.1", + "port": 44338 + }, + "event": { + "dataset": "zookeeper.connection", + "module": "zookeeper", + "duration": 3093417 + }, + "agent": { + "name": "zookeeper-01", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", + "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" + }, + "ecs": { + "version": "1.5.0" } } ``` @@ -120,66 +109,55 @@ An example event for `mntr` looks as following: ```$json { - "_index": "metricbeat-8.0.0-2020.07.06-000001", - "_id": "5aTmJHMB--B0K1AVImYo", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T16:12:08.494Z", - "zookeeper": { - "mntr": { - "open_file_descriptor_count": 49, - "watch_count": 0, - "server_state": "standalone", - "max_file_descriptor_count": 1048576, - "znode_count": 5, - "outstanding_requests": 0, - "ephemerals_count": 0, - "packets": { - "received": 152, - "sent": 151 - }, - "num_alive_connections": 1, - "approximate_data_size": 44, - "latency": { - "max": 0, - "avg": 0, - "min": 0 - } + "@timestamp": "2020-07-06T16:12:08.494Z", + "zookeeper": { + "mntr": { + "open_file_descriptor_count": 49, + "watch_count": 0, + "server_state": "standalone", + "max_file_descriptor_count": 1048576, + "znode_count": 5, + "outstanding_requests": 0, + "ephemerals_count": 0, + "packets": { + "received": 152, + "sent": 151 + }, + "num_alive_connections": 1, + "approximate_data_size": 44, + "latency": { + "max": 0, + "avg": 0, + "min": 0 } - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "zookeeper-01" - }, - "agent": { - "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", - "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651", - "name": "zookeeper-01", - "type": "metricbeat", - "version": "8.0.0" - }, - "service": { - "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653, built on 05/03/2019 12:07 GMT", - "address": "localhost:2181", - "type": "zookeeper" - }, - "event": { - "duration": 15795652, - "dataset": "zookeeper.mntr", - "module": "zookeeper" - }, - "metricset": { - "name": "mntr", - "period": 10000 } }, - "fields": { - "@timestamp": [ - "2020-07-06T16:12:08.494Z" - ] + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "zookeeper-01" + }, + "agent": { + "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", + "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651", + "name": "zookeeper-01", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653, built on 05/03/2019 12:07 GMT", + "address": "localhost:2181", + "type": "zookeeper" + }, + "event": { + "duration": 15795652, + "dataset": "zookeeper.mntr", + "module": "zookeeper" + }, + "metricset": { + "name": "mntr", + "period": 10000 } } ``` @@ -250,66 +228,52 @@ An example event for `server` looks as following: ```$json { - "_index": "metricbeat-8.0.0-2020.07.06-000001", - "_id": "QKTmJHMB--B0K1AVNGfq", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-07-06T16:12:12.409Z", - "event": { - "module": "zookeeper", - "duration": 3001938, - "dataset": "zookeeper.server" - }, - "metricset": { - "name": "server", - "period": 10000 - }, - "ecs": { - "version": "1.5.0" - }, - "host": { - "name": "zookeeper-01" - }, - "agent": { - "name": "zookeeper-01", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", - "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" - }, - "zookeeper": { - "server": { - "zxid": "0x0", - "count": 0, - "version_date": "2019-05-03T12:07:00Z", - "received": 156, - "mode": "standalone", - "latency": { - "avg": 0, - "max": 0, - "min": 0 - }, - "sent": 155, - "epoch": 0, - "node_count": 5, - "connections": 1, - "outstanding": 0 - } - }, - "service": { - "address": "localhost:2181", - "type": "zookeeper", - "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653" + "@timestamp": "2020-07-06T16:12:12.409Z", + "event": { + "module": "zookeeper", + "duration": 3001938, + "dataset": "zookeeper.server" + }, + "metricset": { + "name": "server", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "zookeeper-01" + }, + "agent": { + "name": "zookeeper-01", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "4d221f8f-7147-4855-8ea3-b4d2a5b80ae0", + "id": "2ff8a09c-c7f0-42f2-9fe1-65f7fd460651" + }, + "zookeeper": { + "server": { + "zxid": "0x0", + "count": 0, + "version_date": "2019-05-03T12:07:00Z", + "received": 156, + "mode": "standalone", + "latency": { + "avg": 0, + "max": 0, + "min": 0 + }, + "sent": 155, + "epoch": 0, + "node_count": 5, + "connections": 1, + "outstanding": 0 } }, - "fields": { - "zookeeper.server.version_date": [ - "2019-05-03T12:07:00.000Z" - ], - "@timestamp": [ - "2020-07-06T16:12:12.409Z" - ] + "service": { + "address": "localhost:2181", + "type": "zookeeper", + "version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653" } } ``` From eb92b823aa9a6dd37950e962bde7158f9a6fd024 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:31:01 +0100 Subject: [PATCH 04/18] Adjust zookeeper --- packages/zookeeper/changelog.yml | 5 ++++ .../data_stream/connection/fields/ecs.yml | 15 ++++++++++++ .../zookeeper/data_stream/mntr/fields/ecs.yml | 24 +++++++++---------- .../data_stream/server/fields/ecs.yml | 12 ++++++++++ packages/zookeeper/docs/README.md | 14 ++++++++++- packages/zookeeper/manifest.yml | 2 +- 6 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 packages/zookeeper/data_stream/connection/fields/ecs.yml create mode 100644 packages/zookeeper/data_stream/server/fields/ecs.yml diff --git a/packages/zookeeper/changelog.yml b/packages/zookeeper/changelog.yml index b5d4474f51a..d5e0cfe25e5 100644 --- a/packages/zookeeper/changelog.yml +++ b/packages/zookeeper/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.6" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/zookeeper/data_stream/connection/fields/ecs.yml b/packages/zookeeper/data_stream/connection/fields/ecs.yml new file mode 100644 index 00000000000..1504272c319 --- /dev/null +++ b/packages/zookeeper/data_stream/connection/fields/ecs.yml @@ -0,0 +1,15 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type +- description: IP address of the client. + name: client.ip + type: ip +- description: Port of the client. + name: client.port + type: long diff --git a/packages/zookeeper/data_stream/mntr/fields/ecs.yml b/packages/zookeeper/data_stream/mntr/fields/ecs.yml index 9535a7c27bf..b8bf302ae48 100644 --- a/packages/zookeeper/data_stream/mntr/fields/ecs.yml +++ b/packages/zookeeper/data_stream/mntr/fields/ecs.yml @@ -1,12 +1,12 @@ -- name: service - title: Service - group: 2 - type: group - fields: - - name: version - level: core - type: keyword - description: |- - Version of the service the data was collected from. - This allows to look at a data set only for a specific version of a service. - ignore_above: 1024 +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type +- name: service.version + type: keyword + description: Service version diff --git a/packages/zookeeper/data_stream/server/fields/ecs.yml b/packages/zookeeper/data_stream/server/fields/ecs.yml new file mode 100644 index 00000000000..b8bf302ae48 --- /dev/null +++ b/packages/zookeeper/data_stream/server/fields/ecs.yml @@ -0,0 +1,12 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type +- name: service.version + type: keyword + description: Service version diff --git a/packages/zookeeper/docs/README.md b/packages/zookeeper/docs/README.md index e8766e87f94..f0d4a390e40 100644 --- a/packages/zookeeper/docs/README.md +++ b/packages/zookeeper/docs/README.md @@ -63,6 +63,8 @@ An example event for `connection` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| client.ip | IP address of the client. | ip | +| client.port | Port of the client. | long | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -79,6 +81,7 @@ An example event for `connection` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -95,6 +98,8 @@ An example event for `connection` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | | zookeeper.connection.interest_ops | Interest ops | long | | zookeeper.connection.queued | Queued connections | long | | zookeeper.connection.received | Received connections | long | @@ -183,6 +188,7 @@ An example event for `mntr` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -199,7 +205,9 @@ An example event for `mntr` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | +| service.version | Service version | keyword | | zookeeper.mntr.approximate_data_size | Approximate size of ZooKeeper data. | long | | zookeeper.mntr.ephemerals_count | Number of ephemeral znodes. | long | | zookeeper.mntr.followers | Number of followers seen by the current host. | long | @@ -299,6 +307,7 @@ An example event for `server` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -315,6 +324,9 @@ An example event for `server` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | +| service.version | Service version | keyword | | zookeeper.server.connections | Number of clients currently connected to the server | long | | zookeeper.server.count | Total transactions of the leader in epoch | long | | zookeeper.server.epoch | Epoch value of the Zookeeper transaction ID. An epoch signifies the period in which a server is a leader | long | diff --git a/packages/zookeeper/manifest.yml b/packages/zookeeper/manifest.yml index c4a6d794f9e..07c6b137488 100644 --- a/packages/zookeeper/manifest.yml +++ b/packages/zookeeper/manifest.yml @@ -1,6 +1,6 @@ name: zookeeper title: ZooKeeper -version: 0.2.5 +version: 0.2.6 description: ZooKeeper Integration type: integration icons: From 935ad535ab275712e441145d965cecc0d1eb66c2 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:37:05 +0100 Subject: [PATCH 05/18] Adjust Redis --- packages/redis/changelog.yml | 5 +++ .../redis/data_stream/info/fields/ecs.yml | 24 +++++----- .../redis/data_stream/info/sample_event.json | 10 ----- packages/redis/data_stream/key/fields/ecs.yml | 9 ++++ .../redis/data_stream/key/sample_event.json | 13 ------ .../redis/data_stream/keyspace/fields/ecs.yml | 9 ++++ .../data_stream/keyspace/sample_event.json | 10 ----- packages/redis/docs/README.md | 45 +++++-------------- packages/redis/manifest.yml | 2 +- 9 files changed, 46 insertions(+), 81 deletions(-) create mode 100644 packages/redis/data_stream/key/fields/ecs.yml create mode 100644 packages/redis/data_stream/keyspace/fields/ecs.yml diff --git a/packages/redis/changelog.yml b/packages/redis/changelog.yml index 5431c5f99c1..ab6da8ed0d0 100644 --- a/packages/redis/changelog.yml +++ b/packages/redis/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.7" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/redis/data_stream/info/fields/ecs.yml b/packages/redis/data_stream/info/fields/ecs.yml index 76e4220067b..ec3fca233d2 100644 --- a/packages/redis/data_stream/info/fields/ecs.yml +++ b/packages/redis/data_stream/info/fields/ecs.yml @@ -23,15 +23,15 @@ type: long format: string description: Process id. -- name: service - title: Service - group: 2 - type: group - fields: - - name: version - level: core - type: keyword - description: |- - Version of the service the data was collected from. - This allows to look at a data set only for a specific version of a service. - ignore_above: 1024 +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type +- name: service.version + type: keyword + description: Service version diff --git a/packages/redis/data_stream/info/sample_event.json b/packages/redis/data_stream/info/sample_event.json index 35b616f0b11..86675ad45ba 100644 --- a/packages/redis/data_stream/info/sample_event.json +++ b/packages/redis/data_stream/info/sample_event.json @@ -1,10 +1,5 @@ { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "redis.info", - "namespace": "default", - "type": "metrics" - }, "redis": { "info": { "clients": { @@ -174,11 +169,6 @@ "dataset": "redis.info", "module": "redis" }, - "stream": { - "dataset": "redis.info", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "info", "period": 10000 diff --git a/packages/redis/data_stream/key/fields/ecs.yml b/packages/redis/data_stream/key/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/redis/data_stream/key/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/redis/data_stream/key/sample_event.json b/packages/redis/data_stream/key/sample_event.json index 60765a5e26f..60b9d971e04 100644 --- a/packages/redis/data_stream/key/sample_event.json +++ b/packages/redis/data_stream/key/sample_event.json @@ -1,10 +1,5 @@ { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "redis.key", - "namespace": "default", - "type": "metrics" - }, "redis": { "key": { "expire": { @@ -14,9 +9,6 @@ "length": 3, "name": "foo", "type": "string" - }, - "keyspace": { - "id": "db0" } }, "event": { @@ -24,11 +16,6 @@ "dataset": "redis.key", "module": "redis" }, - "stream": { - "dataset": "redis.key", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "key", "period": 10000 diff --git a/packages/redis/data_stream/keyspace/fields/ecs.yml b/packages/redis/data_stream/keyspace/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/redis/data_stream/keyspace/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/redis/data_stream/keyspace/sample_event.json b/packages/redis/data_stream/keyspace/sample_event.json index 5113ed3fd87..a9dcea122cd 100644 --- a/packages/redis/data_stream/keyspace/sample_event.json +++ b/packages/redis/data_stream/keyspace/sample_event.json @@ -1,10 +1,5 @@ { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "redis.keyspace", - "namespace": "default", - "type": "metrics" - }, "redis": { "keyspace": { "avg_ttl": 359459, @@ -18,11 +13,6 @@ "dataset": "redis.keyspace", "module": "redis" }, - "stream": { - "dataset": "redis.keyspace", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "keyspace", "period": 10000 diff --git a/packages/redis/docs/README.md b/packages/redis/docs/README.md index a605fd3653d..8924e4da04d 100644 --- a/packages/redis/docs/README.md +++ b/packages/redis/docs/README.md @@ -120,11 +120,6 @@ An example event for `info` looks as following: ```$json { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "redis.info", - "namespace": "default", - "type": "metrics" - }, "redis": { "info": { "clients": { @@ -294,11 +289,6 @@ An example event for `info` looks as following: "dataset": "redis.info", "module": "redis" }, - "stream": { - "dataset": "redis.info", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "info", "period": 10000 @@ -334,6 +324,7 @@ An example event for `info` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -458,8 +449,9 @@ An example event for `info` looks as following: | redis.info.stats.sync.full | The number of full resyncs with slaves | long | | redis.info.stats.sync.partial.err | The number of denied partial resync requests | long | | redis.info.stats.sync.partial.ok | The number of accepted partial resync requests | long | -| service.address | Client address | keyword | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | +| service.version | Service version | keyword | ### key @@ -480,11 +472,6 @@ An example event for `key` looks as following: ```$json { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "redis.key", - "namespace": "default", - "type": "metrics" - }, "redis": { "key": { "expire": { @@ -494,9 +481,6 @@ An example event for `key` looks as following: "length": 3, "name": "foo", "type": "string" - }, - "keyspace": { - "id": "db0" } }, "event": { @@ -504,11 +488,6 @@ An example event for `key` looks as following: "dataset": "redis.key", "module": "redis" }, - "stream": { - "dataset": "redis.key", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "key", "period": 10000 @@ -544,6 +523,7 @@ An example event for `key` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -565,6 +545,8 @@ An example event for `key` looks as following: | redis.key.length | Length of the key (Number of elements for lists, length for strings, cardinality for sets). | long | | redis.key.name | Key name. | keyword | | redis.key.type | Key type as shown by `TYPE` command. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### keyspace @@ -577,11 +559,6 @@ An example event for `keyspace` looks as following: ```$json { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "redis.keyspace", - "namespace": "default", - "type": "metrics" - }, "redis": { "keyspace": { "avg_ttl": 359459, @@ -595,11 +572,6 @@ An example event for `keyspace` looks as following: "dataset": "redis.keyspace", "module": "redis" }, - "stream": { - "dataset": "redis.keyspace", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "keyspace", "period": 10000 @@ -635,6 +607,7 @@ An example event for `keyspace` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -655,3 +628,5 @@ An example event for `keyspace` looks as following: | redis.keyspace.expires | | long | | redis.keyspace.id | Keyspace identifier. | keyword | | redis.keyspace.keys | Number of keys in the keyspace. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/redis/manifest.yml b/packages/redis/manifest.yml index 691a39bc139..78e1bf55388 100644 --- a/packages/redis/manifest.yml +++ b/packages/redis/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: redis title: Redis -version: 0.3.6 +version: 0.3.7 license: basic description: Redis Integration type: integration From dade0f9ffe840656453f4c546f241ebb3c0e1855 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:43:40 +0100 Subject: [PATCH 06/18] Adjust Kubernetes integration --- packages/kubernetes/changelog.yml | 5 + .../data_stream/apiserver/sample_event.json | 10 - .../data_stream/container/sample_event.json | 10 - .../controllermanager/sample_event.json | 10 - .../data_stream/event/fields/ecs.yml | 6 + .../data_stream/event/sample_event.json | 10 - .../data_stream/node/sample_event.json | 10 - .../data_stream/pod/sample_event.json | 10 - .../data_stream/proxy/sample_event.json | 10 - .../data_stream/scheduler/sample_event.json | 10 - .../state_container/sample_event.json | 10 - .../state_cronjob/sample_event.json | 10 - .../state_daemonset/sample_event.json | 10 - .../state_deployment/sample_event.json | 10 - .../data_stream/state_node/sample_event.json | 10 - .../state_persistentvolume/sample_event.json | 10 - .../sample_event.json | 10 - .../data_stream/state_pod/sample_event.json | 10 - .../state_replicaset/sample_event.json | 10 - .../state_resourcequota/sample_event.json | 10 - .../state_service/sample_event.json | 10 - .../state_statefulset/sample_event.json | 10 - .../state_storageclass/sample_event.json | 10 - .../data_stream/system/sample_event.json | 10 - .../data_stream/volume/sample_event.json | 10 - packages/kubernetes/docs/README.md | 232 +----------------- packages/kubernetes/manifest.yml | 2 +- 27 files changed, 14 insertions(+), 461 deletions(-) create mode 100644 packages/kubernetes/data_stream/event/fields/ecs.yml diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index 792e1a680dd..e62dd4302b6 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.4.3" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.4.2" changes: - description: Change kibana.version constraint to be more conservative. diff --git a/packages/kubernetes/data_stream/apiserver/sample_event.json b/packages/kubernetes/data_stream/apiserver/sample_event.json index e17aa1b2baf..fda7d3222a7 100644 --- a/packages/kubernetes/data_stream/apiserver/sample_event.json +++ b/packages/kubernetes/data_stream/apiserver/sample_event.json @@ -38,16 +38,6 @@ "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "type": "metricbeat" }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.apiserver" - }, - "stream": { - "dataset": "kubernetes.apiserver", - "namespace": "default", - "type": "metrics" - }, "host": { "id": "b0e83d397c054b8a99a431072fe4617b", "containerized": false, diff --git a/packages/kubernetes/data_stream/container/sample_event.json b/packages/kubernetes/data_stream/container/sample_event.json index 1ae43a4470f..2bbe7c0a936 100644 --- a/packages/kubernetes/data_stream/container/sample_event.json +++ b/packages/kubernetes/data_stream/container/sample_event.json @@ -79,16 +79,6 @@ } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.container" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.container" - }, "host": { "containerized": false, "ip": [ diff --git a/packages/kubernetes/data_stream/controllermanager/sample_event.json b/packages/kubernetes/data_stream/controllermanager/sample_event.json index 9d1b4207a45..7aa39a4dd4b 100644 --- a/packages/kubernetes/data_stream/controllermanager/sample_event.json +++ b/packages/kubernetes/data_stream/controllermanager/sample_event.json @@ -27,11 +27,6 @@ "module": "kubernetes", "duration": 8893806 }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.controllermanager" - }, "ecs": { "version": "1.5.0" }, @@ -94,10 +89,5 @@ "service": { "address": "localhost:10252", "type": "kubernetes" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.controllermanager" } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/event/fields/ecs.yml b/packages/kubernetes/data_stream/event/fields/ecs.yml new file mode 100644 index 00000000000..7509bf59910 --- /dev/null +++ b/packages/kubernetes/data_stream/event/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.type + type: keyword + description: Service type diff --git a/packages/kubernetes/data_stream/event/sample_event.json b/packages/kubernetes/data_stream/event/sample_event.json index 8226317aa9c..01c0c93fcfd 100644 --- a/packages/kubernetes/data_stream/event/sample_event.json +++ b/packages/kubernetes/data_stream/event/sample_event.json @@ -3,11 +3,6 @@ "metricset": { "name": "event" }, - "stream": { - "dataset": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -59,11 +54,6 @@ } } }, - "dataset": { - "name": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, "host": { "id": "b0e83d397c054b8a99a431072fe4617b", "containerized": false, diff --git a/packages/kubernetes/data_stream/node/sample_event.json b/packages/kubernetes/data_stream/node/sample_event.json index 1ef8b7d994b..3128c86a50b 100644 --- a/packages/kubernetes/data_stream/node/sample_event.json +++ b/packages/kubernetes/data_stream/node/sample_event.json @@ -135,16 +135,6 @@ } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.node" - }, "agent": { "name": "minikube", "type": "metricbeat", diff --git a/packages/kubernetes/data_stream/pod/sample_event.json b/packages/kubernetes/data_stream/pod/sample_event.json index 857b5e83245..62e44029dac 100644 --- a/packages/kubernetes/data_stream/pod/sample_event.json +++ b/packages/kubernetes/data_stream/pod/sample_event.json @@ -59,11 +59,6 @@ "dataset": "kubernetes.pod", "module": "kubernetes" }, - "stream": { - "dataset": "kubernetes.pod", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, @@ -75,11 +70,6 @@ "type": "kubernetes", "address": "minikube:10250" }, - "dataset": { - "type": "metrics", - "name": "kubernetes.pod", - "namespace": "default" - }, "host": { "name": "minikube", "hostname": "minikube", diff --git a/packages/kubernetes/data_stream/proxy/sample_event.json b/packages/kubernetes/data_stream/proxy/sample_event.json index 30074f2136a..50ba492ea8b 100644 --- a/packages/kubernetes/data_stream/proxy/sample_event.json +++ b/packages/kubernetes/data_stream/proxy/sample_event.json @@ -7,16 +7,6 @@ "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.proxy" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.proxy", - "namespace": "default" - }, "host": { "ip": [ "192.168.64.10", diff --git a/packages/kubernetes/data_stream/scheduler/sample_event.json b/packages/kubernetes/data_stream/scheduler/sample_event.json index a0a2cff70d1..b932b1797d4 100644 --- a/packages/kubernetes/data_stream/scheduler/sample_event.json +++ b/packages/kubernetes/data_stream/scheduler/sample_event.json @@ -7,16 +7,6 @@ "name": "minikube", "type": "metricbeat" }, - "dataset": { - "name": "kubernetes.scheduler", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.scheduler" - }, "host": { "hostname": "minikube", "architecture": "x86_64", diff --git a/packages/kubernetes/data_stream/state_container/sample_event.json b/packages/kubernetes/data_stream/state_container/sample_event.json index 644844c64c8..fda9be93103 100644 --- a/packages/kubernetes/data_stream/state_container/sample_event.json +++ b/packages/kubernetes/data_stream/state_container/sample_event.json @@ -55,16 +55,6 @@ }, "namespace": "kube-system" }, - "dataset": { - "name": "kubernetes.state_container", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_container", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/kubernetes/data_stream/state_cronjob/sample_event.json b/packages/kubernetes/data_stream/state_cronjob/sample_event.json index 3ed3de3c134..f297190ba72 100644 --- a/packages/kubernetes/data_stream/state_cronjob/sample_event.json +++ b/packages/kubernetes/data_stream/state_cronjob/sample_event.json @@ -56,16 +56,6 @@ } } }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_cronjob", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_cronjob" - }, "agent": { "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", diff --git a/packages/kubernetes/data_stream/state_daemonset/sample_event.json b/packages/kubernetes/data_stream/state_daemonset/sample_event.json index 248cc32142a..54b75c87c93 100644 --- a/packages/kubernetes/data_stream/state_daemonset/sample_event.json +++ b/packages/kubernetes/data_stream/state_daemonset/sample_event.json @@ -31,16 +31,6 @@ }, "namespace": "kube-system" }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_daemonset", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_daemonset", - "namespace": "default" - }, "host": { "mac": [ "02:42:ac:11:00:0b" diff --git a/packages/kubernetes/data_stream/state_deployment/sample_event.json b/packages/kubernetes/data_stream/state_deployment/sample_event.json index a7d288bf9da..fd79bacedd1 100644 --- a/packages/kubernetes/data_stream/state_deployment/sample_event.json +++ b/packages/kubernetes/data_stream/state_deployment/sample_event.json @@ -32,16 +32,6 @@ }, "namespace": "kube-system" }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_deployment", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_deployment", - "namespace": "default" - }, "host": { "mac": [ "02:42:ac:11:00:0b" diff --git a/packages/kubernetes/data_stream/state_node/sample_event.json b/packages/kubernetes/data_stream/state_node/sample_event.json index 7121a3ed4fa..9b7f451ab60 100644 --- a/packages/kubernetes/data_stream/state_node/sample_event.json +++ b/packages/kubernetes/data_stream/state_node/sample_event.json @@ -24,16 +24,6 @@ "172.17.0.11" ] }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_node" - }, "metricset": { "name": "state_node", "period": 10000 diff --git a/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json b/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json index cfcb154d9eb..cbc6f7f8467 100644 --- a/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json +++ b/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json @@ -28,16 +28,6 @@ "type": "local" } }, - "dataset": { - "name": "kubernetes.state_persistentvolume", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolume", - "namespace": "default" - }, "host": { "ip": [ "172.17.0.11" diff --git a/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json b/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json index 6d63fa4e49f..0a1204964ae 100644 --- a/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json +++ b/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json @@ -26,11 +26,6 @@ "access_mode": "ReadWriteOnce" } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_persistentvolumeclaim" - }, "agent": { "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "type": "metricbeat", @@ -38,11 +33,6 @@ "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolumeclaim", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/kubernetes/data_stream/state_pod/sample_event.json b/packages/kubernetes/data_stream/state_pod/sample_event.json index 82194c27b65..1b868887bd3 100644 --- a/packages/kubernetes/data_stream/state_pod/sample_event.json +++ b/packages/kubernetes/data_stream/state_pod/sample_event.json @@ -1,15 +1,5 @@ { "@timestamp": "2020-06-25T12:38:34.469Z", - "dataset": { - "name": "kubernetes.state_pod", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_pod" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/kubernetes/data_stream/state_replicaset/sample_event.json b/packages/kubernetes/data_stream/state_replicaset/sample_event.json index 6cc5548f5a7..e5506863d77 100644 --- a/packages/kubernetes/data_stream/state_replicaset/sample_event.json +++ b/packages/kubernetes/data_stream/state_replicaset/sample_event.json @@ -8,16 +8,6 @@ "period": 10000, "name": "state_replicaset" }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_replicaset" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_replicaset", - "namespace": "default" - }, "event": { "module": "kubernetes", "duration": 5456128, diff --git a/packages/kubernetes/data_stream/state_resourcequota/sample_event.json b/packages/kubernetes/data_stream/state_resourcequota/sample_event.json index 73a1813a8fd..d3603943f0a 100644 --- a/packages/kubernetes/data_stream/state_resourcequota/sample_event.json +++ b/packages/kubernetes/data_stream/state_resourcequota/sample_event.json @@ -4,11 +4,6 @@ "name": "state_resourcequota", "period": 10000 }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_resourcequota", - "namespace": "default" - }, "host": { "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", @@ -57,10 +52,5 @@ "type": "hard", "quota": 1 } - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_resourcequota", - "namespace": "default" } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_service/sample_event.json b/packages/kubernetes/data_stream/state_service/sample_event.json index 13fcd4e303e..a6cd05ed179 100644 --- a/packages/kubernetes/data_stream/state_service/sample_event.json +++ b/packages/kubernetes/data_stream/state_service/sample_event.json @@ -58,15 +58,5 @@ "service": { "address": "kube-state-metrics:8080", "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_statefulset/sample_event.json b/packages/kubernetes/data_stream/state_statefulset/sample_event.json index 16afea87e60..9a87bc3df2d 100644 --- a/packages/kubernetes/data_stream/state_statefulset/sample_event.json +++ b/packages/kubernetes/data_stream/state_statefulset/sample_event.json @@ -58,15 +58,5 @@ "service": { "address": "kube-state-metrics:8080", "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" } } \ No newline at end of file diff --git a/packages/kubernetes/data_stream/state_storageclass/sample_event.json b/packages/kubernetes/data_stream/state_storageclass/sample_event.json index 1f5d878752b..de074d381d9 100644 --- a/packages/kubernetes/data_stream/state_storageclass/sample_event.json +++ b/packages/kubernetes/data_stream/state_storageclass/sample_event.json @@ -19,16 +19,6 @@ "addonmanager_kubernetes_io_mode": "EnsureExists" } }, - "dataset": { - "name": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, "host": { "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "architecture": "x86_64", diff --git a/packages/kubernetes/data_stream/system/sample_event.json b/packages/kubernetes/data_stream/system/sample_event.json index 59ed7ff327a..1c490ef754c 100644 --- a/packages/kubernetes/data_stream/system/sample_event.json +++ b/packages/kubernetes/data_stream/system/sample_event.json @@ -1,10 +1,5 @@ { "@timestamp": "2020-06-25T12:39:59.647Z", - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.system" - }, "service": { "address": "minikube:10250", "type": "kubernetes" @@ -14,11 +9,6 @@ "dataset": "kubernetes.system", "module": "kubernetes" }, - "stream": { - "dataset": "kubernetes.system", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/kubernetes/data_stream/volume/sample_event.json b/packages/kubernetes/data_stream/volume/sample_event.json index 859a01a72ee..700d089f1bd 100644 --- a/packages/kubernetes/data_stream/volume/sample_event.json +++ b/packages/kubernetes/data_stream/volume/sample_event.json @@ -39,16 +39,6 @@ "name": "minikube" } }, - "dataset": { - "type": "metrics", - "name": "kubernetes.volume", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.volume" - }, "host": { "architecture": "x86_64", "os": { diff --git a/packages/kubernetes/docs/README.md b/packages/kubernetes/docs/README.md index 1483d52e86f..b7a8c4c3da6 100644 --- a/packages/kubernetes/docs/README.md +++ b/packages/kubernetes/docs/README.md @@ -153,16 +153,6 @@ An example event for `apiserver` looks as following: "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "type": "metricbeat" }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.apiserver" - }, - "stream": { - "dataset": "kubernetes.apiserver", - "namespace": "default", - "type": "metrics" - }, "host": { "id": "b0e83d397c054b8a99a431072fe4617b", "containerized": false, @@ -361,16 +351,6 @@ An example event for `container` looks as following: } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.container" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.container" - }, "host": { "containerized": false, "ip": [ @@ -556,11 +536,6 @@ An example event for `controllermanager` looks as following: "module": "kubernetes", "duration": 8893806 }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.controllermanager" - }, "ecs": { "version": "1.5.0" }, @@ -623,11 +598,6 @@ An example event for `controllermanager` looks as following: "service": { "address": "localhost:10252", "type": "kubernetes" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.controllermanager" } } ``` @@ -719,11 +689,6 @@ An example event for `event` looks as following: "metricset": { "name": "event" }, - "stream": { - "dataset": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -775,11 +740,6 @@ An example event for `event` looks as following: } } }, - "dataset": { - "name": "kubernetes.event", - "namespace": "default", - "type": "metrics" - }, "host": { "id": "b0e83d397c054b8a99a431072fe4617b", "containerized": false, @@ -825,6 +785,7 @@ An example event for `event` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -861,6 +822,7 @@ An example event for `event` looks as following: | kubernetes.event.timestamp.first_occurrence | Timestamp of first occurrence of event | date | | kubernetes.event.timestamp.last_occurrence | Timestamp of last occurrence of event | date | | kubernetes.event.type | Type of the given event | keyword | +| service.type | Service type | keyword | ### node @@ -1008,16 +970,6 @@ An example event for `node` looks as following: } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.node" - }, "agent": { "name": "minikube", "type": "metricbeat", @@ -1176,11 +1128,6 @@ An example event for `pod` looks as following: "dataset": "kubernetes.pod", "module": "kubernetes" }, - "stream": { - "dataset": "kubernetes.pod", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, @@ -1192,11 +1139,6 @@ An example event for `pod` looks as following: "type": "kubernetes", "address": "minikube:10250" }, - "dataset": { - "type": "metrics", - "name": "kubernetes.pod", - "namespace": "default" - }, "host": { "name": "minikube", "hostname": "minikube", @@ -1339,16 +1281,6 @@ An example event for `proxy` looks as following: "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.proxy" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.proxy", - "namespace": "default" - }, "host": { "ip": [ "192.168.64.10", @@ -1643,16 +1575,6 @@ An example event for `scheduler` looks as following: "name": "minikube", "type": "metricbeat" }, - "dataset": { - "name": "kubernetes.scheduler", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.scheduler" - }, "host": { "hostname": "minikube", "architecture": "x86_64", @@ -1866,16 +1788,6 @@ An example event for `state_container` looks as following: }, "namespace": "kube-system" }, - "dataset": { - "name": "kubernetes.state_container", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_container", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -2033,16 +1945,6 @@ An example event for `state_cronjob` looks as following: } } }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_cronjob", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_cronjob" - }, "agent": { "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", @@ -2157,16 +2059,6 @@ An example event for `state_daemonset` looks as following: }, "namespace": "kube-system" }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_daemonset", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_daemonset", - "namespace": "default" - }, "host": { "mac": [ "02:42:ac:11:00:0b" @@ -2299,16 +2191,6 @@ An example event for `state_deployment` looks as following: }, "namespace": "kube-system" }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_deployment", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_deployment", - "namespace": "default" - }, "host": { "mac": [ "02:42:ac:11:00:0b" @@ -2433,16 +2315,6 @@ An example event for `state_node` looks as following: "172.17.0.11" ] }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_node" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_node" - }, "metricset": { "name": "state_node", "period": 10000 @@ -2611,16 +2483,6 @@ An example event for `state_persistentvolume` looks as following: "type": "local" } }, - "dataset": { - "name": "kubernetes.state_persistentvolume", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolume", - "namespace": "default" - }, "host": { "ip": [ "172.17.0.11" @@ -2747,11 +2609,6 @@ An example event for `state_persistentvolumeclaim` looks as following: "access_mode": "ReadWriteOnce" } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_persistentvolumeclaim" - }, "agent": { "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "type": "metricbeat", @@ -2759,11 +2616,6 @@ An example event for `state_persistentvolumeclaim` looks as following: "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_persistentvolumeclaim", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -2861,16 +2713,6 @@ An example event for `state_pod` looks as following: ```$json { "@timestamp": "2020-06-25T12:38:34.469Z", - "dataset": { - "name": "kubernetes.state_pod", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.state_pod" - }, "ecs": { "version": "1.5.0" }, @@ -3018,16 +2860,6 @@ An example event for `state_replicaset` looks as following: "period": 10000, "name": "state_replicaset" }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.state_replicaset" - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_replicaset", - "namespace": "default" - }, "event": { "module": "kubernetes", "duration": 5456128, @@ -3162,11 +2994,6 @@ An example event for `state_resourcequota` looks as following: "name": "state_resourcequota", "period": 10000 }, - "dataset": { - "type": "metrics", - "name": "kubernetes.state_resourcequota", - "namespace": "default" - }, "host": { "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", @@ -3215,11 +3042,6 @@ An example event for `state_resourcequota` looks as following: "type": "hard", "quota": 1 } - }, - "stream": { - "type": "metrics", - "dataset": "kubernetes.state_resourcequota", - "namespace": "default" } } ``` @@ -3351,16 +3173,6 @@ An example event for `state_service` looks as following: "service": { "address": "kube-state-metrics:8080", "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_service", - "namespace": "default", - "type": "metrics" } } ``` @@ -3495,16 +3307,6 @@ An example event for `state_statefulset` looks as following: "service": { "address": "kube-state-metrics:8080", "type": "kubernetes" - }, - "dataset": { - "name": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_statefulset", - "namespace": "default", - "type": "metrics" } } ``` @@ -3597,16 +3399,6 @@ An example event for `state_storageclass` looks as following: "addonmanager_kubernetes_io_mode": "EnsureExists" } }, - "dataset": { - "name": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "kubernetes.state_storageclass", - "namespace": "default", - "type": "metrics" - }, "host": { "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", "architecture": "x86_64", @@ -3716,11 +3508,6 @@ An example event for `system` looks as following: ```$json { "@timestamp": "2020-06-25T12:39:59.647Z", - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "kubernetes.system" - }, "service": { "address": "minikube:10250", "type": "kubernetes" @@ -3730,11 +3517,6 @@ An example event for `system` looks as following: "dataset": "kubernetes.system", "module": "kubernetes" }, - "stream": { - "dataset": "kubernetes.system", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, @@ -3939,16 +3721,6 @@ An example event for `volume` looks as following: "name": "minikube" } }, - "dataset": { - "type": "metrics", - "name": "kubernetes.volume", - "namespace": "default" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "kubernetes.volume" - }, "host": { "architecture": "x86_64", "os": { diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index 3f0d41cca13..c852b0e2fb2 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kubernetes title: Kubernetes -version: 0.4.2 +version: 0.4.3 license: basic description: Kubernetes Integration type: integration From 8db3bc3ec557f533d475e1237b7d6e78e21896c6 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 1 Mar 2021 17:50:13 +0100 Subject: [PATCH 07/18] Correct Azure integration --- packages/azure/changelog.yml | 5 ++ .../activitylogs/sample_event.json | 21 +----- .../data_stream/auditlogs/sample_event.json | 17 ----- .../platformlogs/sample_event.json | 15 ----- .../data_stream/signinlogs/sample_event.json | 13 ---- packages/azure/docs/README.md | 66 +------------------ packages/azure/manifest.yml | 2 +- 7 files changed, 8 insertions(+), 131 deletions(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 88ef1801486..354e2f3058d 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.1" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.2.0" changes: - description: Add changes to use ECS 1.8 fields. diff --git a/packages/azure/data_stream/activitylogs/sample_event.json b/packages/azure/data_stream/activitylogs/sample_event.json index 799fe904332..9272f44ffc4 100644 --- a/packages/azure/data_stream/activitylogs/sample_event.json +++ b/packages/azure/data_stream/activitylogs/sample_event.json @@ -2,22 +2,9 @@ "log": { "level": "Information" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-activity-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-02T08:51:36.997Z", "ecs": { "version": "1.5.0" @@ -94,13 +81,7 @@ }, "category": "Administrative", "event_category": "Administrative", - "result_signature": "Succeeded.", - "properties": { - "eventCategory": "Administrative", - "hierarchy": "", - "message": "Microsoft.Resources/deployments/write", - "entity": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace" - } + "result_signature": "Succeeded." } } } \ No newline at end of file diff --git a/packages/azure/data_stream/auditlogs/sample_event.json b/packages/azure/data_stream/auditlogs/sample_event.json index c57595de91d..04fabb65f3f 100644 --- a/packages/azure/data_stream/auditlogs/sample_event.json +++ b/packages/azure/data_stream/auditlogs/sample_event.json @@ -2,22 +2,9 @@ "log": { "level": "Information" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-auditlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-02T08:51:36.997Z", "ecs": { "version": "1.5.0" @@ -48,18 +35,14 @@ "azure.auditlogs.properties.category": "Device", "azure.auditlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "azure.auditlogs.properties.id": "Directory_ESQ", - "azure.auditlogs.properties.initiated_by.app.appId": null, "azure.auditlogs.properties.initiated_by.app.displayName": "Device Registration Service", "azure.auditlogs.properties.initiated_by.app.servicePrincipalId": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.initiated_by.app.servicePrincipalName": null, "azure.auditlogs.properties.logged_by_service": "Core Directory", "azure.auditlogs.properties.operation_type": "Update", "azure.auditlogs.properties.result_reason": "", "azure.auditlogs.properties.target_resources.0.display_name": "LAPTOP-12", "azure.auditlogs.properties.target_resources.0.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.display_name": "Included Updated Properties", "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "\"\"", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": null, "azure.auditlogs.properties.target_resources.0.type": "Device", "azure.auditlogs.result_signature": "None" } \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/sample_event.json b/packages/azure/data_stream/platformlogs/sample_event.json index 85752e6fa49..44c25798414 100644 --- a/packages/azure/data_stream/platformlogs/sample_event.json +++ b/packages/azure/data_stream/platformlogs/sample_event.json @@ -12,23 +12,10 @@ "version": "7.10.0", "snapshot": false }, - "azure-eventhub": { - "sequence_number": 15, - "consumer_group": "$Default", - "offset": 4294976088, - "eventhub": "insights-logs-operationallogs", - "enqueued_time": "2020-11-05T14:08:28.137Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure", "region": "West Europe" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-05T14:07:32.000Z", "ecs": { "version": "1.5.0" @@ -51,8 +38,6 @@ "azure": { "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", "platformlogs": { - "Status": "Succeeded", - "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", "Caller": "Portal", "ActivityId": "5890c6fc-fc6b-47cd-971a-2366a1641d99", "EventTimeString": "11/5/2020 2:07:32 PM +00:00", diff --git a/packages/azure/data_stream/signinlogs/sample_event.json b/packages/azure/data_stream/signinlogs/sample_event.json index 4aa0d3b2abb..0661c8c73fc 100644 --- a/packages/azure/data_stream/signinlogs/sample_event.json +++ b/packages/azure/data_stream/signinlogs/sample_event.json @@ -2,22 +2,9 @@ "log": { "level": "Information" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-signinlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-02T08:51:36.997Z", "ecs": { "version": "1.5.0" diff --git a/packages/azure/docs/README.md b/packages/azure/docs/README.md index 2e4949d1943..6e61763647f 100644 --- a/packages/azure/docs/README.md +++ b/packages/azure/docs/README.md @@ -65,22 +65,9 @@ An example event for `activitylogs` looks as following: "log": { "level": "Information" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-activity-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-02T08:51:36.997Z", "ecs": { "version": "1.5.0" @@ -157,13 +144,7 @@ An example event for `activitylogs` looks as following: }, "category": "Administrative", "event_category": "Administrative", - "result_signature": "Succeeded.", - "properties": { - "eventCategory": "Administrative", - "hierarchy": "", - "message": "Microsoft.Resources/deployments/write", - "entity": "/subscriptions/3f041b6d-fc31-41d8-8ff6-e5f16e6747ff/resourceGroups/obs-test/providers/Microsoft.Resources/deployments/NoMarketplace" - } + "result_signature": "Succeeded." } } } @@ -310,23 +291,10 @@ An example event for `platformlogs` looks as following: "version": "7.10.0", "snapshot": false }, - "azure-eventhub": { - "sequence_number": 15, - "consumer_group": "$Default", - "offset": 4294976088, - "eventhub": "insights-logs-operationallogs", - "enqueued_time": "2020-11-05T14:08:28.137Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure", "region": "West Europe" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-05T14:07:32.000Z", "ecs": { "version": "1.5.0" @@ -349,8 +317,6 @@ An example event for `platformlogs` looks as following: "azure": { "subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", "platformlogs": { - "Status": "Succeeded", - "SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", "Caller": "Portal", "ActivityId": "5890c6fc-fc6b-47cd-971a-2366a1641d99", "EventTimeString": "11/5/2020 2:07:32 PM +00:00", @@ -498,22 +464,9 @@ An example event for `auditlogs` looks as following: "log": { "level": "Information" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-auditlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-02T08:51:36.997Z", "ecs": { "version": "1.5.0" @@ -544,18 +497,14 @@ An example event for `auditlogs` looks as following: "azure.auditlogs.properties.category": "Device", "azure.auditlogs.properties.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "azure.auditlogs.properties.id": "Directory_ESQ", - "azure.auditlogs.properties.initiated_by.app.appId": null, "azure.auditlogs.properties.initiated_by.app.displayName": "Device Registration Service", "azure.auditlogs.properties.initiated_by.app.servicePrincipalId": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.initiated_by.app.servicePrincipalName": null, "azure.auditlogs.properties.logged_by_service": "Core Directory", "azure.auditlogs.properties.operation_type": "Update", "azure.auditlogs.properties.result_reason": "", "azure.auditlogs.properties.target_resources.0.display_name": "LAPTOP-12", "azure.auditlogs.properties.target_resources.0.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.display_name": "Included Updated Properties", "azure.auditlogs.properties.target_resources.0.modified_properties.0.new_value": "\"\"", - "azure.auditlogs.properties.target_resources.0.modified_properties.0.old_value": null, "azure.auditlogs.properties.target_resources.0.type": "Device", "azure.auditlogs.result_signature": "None" } @@ -701,22 +650,9 @@ An example event for `signinlogs` looks as following: "log": { "level": "Information" }, - "azure-eventhub": { - "sequence_number": 643, - "consumer_group": "$Default", - "offset": 107374182400, - "eventhub": "insights-signinlogs-logs", - "enqueued_time": "2020-11-02T08:59:38.905Z" - }, - "tags": [ - "forwarded" - ], "cloud": { "provider": "azure" }, - "input": { - "type": "azure-eventhub" - }, "@timestamp": "2020-11-02T08:51:36.997Z", "ecs": { "version": "1.5.0" diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index f41ec799aef..1a602430d50 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure -version: 0.2.0 +version: 0.2.1 release: beta description: Azure Integration type: integration From cb2faed81ac68d8f790cc8031e2f681016ee2d15 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:06:38 +0100 Subject: [PATCH 08/18] Fix Haproxy --- packages/haproxy/changelog.yml | 5 +++++ packages/haproxy/data_stream/info/fields/ecs.yml | 6 ++++++ packages/haproxy/data_stream/stat/fields/ecs.yml | 6 ++++++ packages/haproxy/docs/README.md | 4 ++++ packages/haproxy/manifest.yml | 2 +- 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/haproxy/changelog.yml b/packages/haproxy/changelog.yml index 94cc53631f8..a1a83633824 100644 --- a/packages/haproxy/changelog.yml +++ b/packages/haproxy/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.7" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/haproxy/data_stream/info/fields/ecs.yml b/packages/haproxy/data_stream/info/fields/ecs.yml index 397e9478a2c..9d86a4461fd 100644 --- a/packages/haproxy/data_stream/info/fields/ecs.yml +++ b/packages/haproxy/data_stream/info/fields/ecs.yml @@ -8,3 +8,9 @@ type: long format: string description: Process id. +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/haproxy/data_stream/stat/fields/ecs.yml b/packages/haproxy/data_stream/stat/fields/ecs.yml index 397e9478a2c..9d86a4461fd 100644 --- a/packages/haproxy/data_stream/stat/fields/ecs.yml +++ b/packages/haproxy/data_stream/stat/fields/ecs.yml @@ -8,3 +8,9 @@ type: long format: string description: Process id. +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/haproxy/docs/README.md b/packages/haproxy/docs/README.md index 07ada5379f3..cef29f84b93 100644 --- a/packages/haproxy/docs/README.md +++ b/packages/haproxy/docs/README.md @@ -313,6 +313,8 @@ The fields reported are: | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | process.pid | Process id. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### stat @@ -536,4 +538,6 @@ The fields reported are: | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | process.pid | Process id. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/haproxy/manifest.yml b/packages/haproxy/manifest.yml index e78ee8042ae..68b004d55d4 100644 --- a/packages/haproxy/manifest.yml +++ b/packages/haproxy/manifest.yml @@ -1,6 +1,6 @@ name: haproxy title: HAProxy -version: 0.2.6 +version: 0.2.7 description: HAProxy Integration type: integration icons: From 1396bbfc4fb4527ef6eedf21832f6435f9eeba37 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:09:13 +0100 Subject: [PATCH 09/18] Fix nats --- packages/nats/changelog.yml | 5 +++++ packages/nats/data_stream/log/sample_event.json | 4 ++-- packages/nats/docs/README.md | 4 ++-- packages/nats/manifest.yml | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/nats/changelog.yml b/packages/nats/changelog.yml index 60cf0dfda4b..2d17c0fecb9 100644 --- a/packages/nats/changelog.yml +++ b/packages/nats/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.2" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.1" changes: - description: Change kibana.version constraint to be more conservative. diff --git a/packages/nats/data_stream/log/sample_event.json b/packages/nats/data_stream/log/sample_event.json index ce4ee4ddb54..194839a89e8 100644 --- a/packages/nats/data_stream/log/sample_event.json +++ b/packages/nats/data_stream/log/sample_event.json @@ -18,7 +18,7 @@ "version": "7.11.0" }, "process": { - "pid": "6" + "pid": 6 }, "log": { "file": { @@ -74,7 +74,7 @@ "architecture": "x86_64" }, "client": { - "port": "53482", + "port": 53482, "ip": "192.168.192.3" }, "event": { diff --git a/packages/nats/docs/README.md b/packages/nats/docs/README.md index 7d5926de73c..efc9c21a37d 100644 --- a/packages/nats/docs/README.md +++ b/packages/nats/docs/README.md @@ -37,7 +37,7 @@ An example event for `log` looks as following: "version": "7.11.0" }, "process": { - "pid": "6" + "pid": 6 }, "log": { "file": { @@ -93,7 +93,7 @@ An example event for `log` looks as following: "architecture": "x86_64" }, "client": { - "port": "53482", + "port": 53482, "ip": "192.168.192.3" }, "event": { diff --git a/packages/nats/manifest.yml b/packages/nats/manifest.yml index 7eb99344163..6d64abab00c 100644 --- a/packages/nats/manifest.yml +++ b/packages/nats/manifest.yml @@ -1,6 +1,6 @@ name: nats title: NATS -version: 0.1.1 +version: 0.1.2 release: experimental description: NATS Integration type: integration From 601587e6ae7256a67537b8f82fd03dafb0c1b53b Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:26:57 +0100 Subject: [PATCH 10/18] Fix IIS --- packages/iis/changelog.yml | 5 ++ .../iis/data_stream/access/fields/ecs.yml | 3 + .../iis/data_stream/access/sample_event.json | 14 ----- .../application_pool/fields/ecs.yml | 9 +++ .../application_pool/sample_event.json | 8 --- packages/iis/data_stream/error/fields/ecs.yml | 3 + .../iis/data_stream/error/sample_event.json | 14 ----- .../iis/data_stream/webserver/fields/ecs.yml | 9 +++ .../data_stream/webserver/sample_event.json | 9 +-- .../iis/data_stream/website/fields/ecs.yml | 6 ++ .../iis/data_stream/website/sample_event.json | 5 -- packages/iis/docs/README.md | 60 ++++--------------- packages/iis/manifest.yml | 2 +- 13 files changed, 48 insertions(+), 99 deletions(-) create mode 100644 packages/iis/data_stream/application_pool/fields/ecs.yml create mode 100644 packages/iis/data_stream/webserver/fields/ecs.yml create mode 100644 packages/iis/data_stream/website/fields/ecs.yml diff --git a/packages/iis/changelog.yml b/packages/iis/changelog.yml index 26ef03b0f53..0ce9c4e983a 100644 --- a/packages/iis/changelog.yml +++ b/packages/iis/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.1" + changes: + - description: Use correct types for `source.port` and `source.ip` + type: bugfix + link: https://github.com/elastic/integrations/pull/737 - version: "0.1.0" changes: - description: initial release diff --git a/packages/iis/data_stream/access/fields/ecs.yml b/packages/iis/data_stream/access/fields/ecs.yml index 38caaf8e940..b056043d0a1 100644 --- a/packages/iis/data_stream/access/fields/ecs.yml +++ b/packages/iis/data_stream/access/fields/ecs.yml @@ -238,3 +238,6 @@ - name: os.version type: keyword description: Operating system version. +- name: ecs.version + type: keyword + description: ECS version diff --git a/packages/iis/data_stream/access/sample_event.json b/packages/iis/data_stream/access/sample_event.json index e63d382f3f2..29281518ba5 100644 --- a/packages/iis/data_stream/access/sample_event.json +++ b/packages/iis/data_stream/access/sample_event.json @@ -7,12 +7,6 @@ "version": "8.0.0" }, "temp": {}, - "log": { - "file": { - "path": "C:\\inetpub\\logs\\LogFiles\\W3SVC2\\u_ex181119.log" - }, - "offset": 261 - }, "destination": { "address": "127.0.0.1", "port": 80, @@ -25,9 +19,6 @@ "url": { "path": "/" }, - "input": { - "type": "log" - }, "iis": { "access": { "sub_status": 3, @@ -65,11 +56,6 @@ ], "outcome": "failure" }, - "dataset": { - "name": "iis.access", - "namespace": "default", - "type": "logs" - }, "user_agent": { "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "os": { diff --git a/packages/iis/data_stream/application_pool/fields/ecs.yml b/packages/iis/data_stream/application_pool/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/iis/data_stream/application_pool/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/iis/data_stream/application_pool/sample_event.json b/packages/iis/data_stream/application_pool/sample_event.json index 6b8c65ad9d7..6bd091a989c 100644 --- a/packages/iis/data_stream/application_pool/sample_event.json +++ b/packages/iis/data_stream/application_pool/sample_event.json @@ -1,8 +1,5 @@ { "@timestamp": "2020-07-08T11:41:31.048Z", - "process": { - "pid": 51224 - }, "event": { "dataset": "iis.application_pool", "module": "iis", @@ -37,10 +34,5 @@ "metricset": { "period": 10000, "name": "application_pool" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "iis.application_pool" } } \ No newline at end of file diff --git a/packages/iis/data_stream/error/fields/ecs.yml b/packages/iis/data_stream/error/fields/ecs.yml index d0c38cbc4c1..8f6f97298b3 100644 --- a/packages/iis/data_stream/error/fields/ecs.yml +++ b/packages/iis/data_stream/error/fields/ecs.yml @@ -144,3 +144,6 @@ - name: message description: Message type: text +- name: ecs.version + type: keyword + description: ECS version diff --git a/packages/iis/data_stream/error/sample_event.json b/packages/iis/data_stream/error/sample_event.json index 6bee688ba3f..d06ead836ef 100644 --- a/packages/iis/data_stream/error/sample_event.json +++ b/packages/iis/data_stream/error/sample_event.json @@ -6,12 +6,6 @@ "ephemeral_id": "3f65b650-b6a3-4694-83b3-0c324a60809d", "version": "8.0.0" }, - "log": { - "file": { - "path": "c:\\Windows\\System32\\LogFiles\\HTTPERR\\httperr1.log" - }, - "offset": 199 - }, "destination": { "address": "::1%0", "port": 80, @@ -22,9 +16,6 @@ "port": 59827, "ip": "::1" }, - "input": { - "type": "log" - }, "iis": { "error": { "reason_phrase": "Timer_ConnectionIdle" @@ -50,10 +41,5 @@ "type": [ "connection" ] - }, - "dataset": { - "name": "iis.error", - "namespace": "default", - "type": "logs" } } \ No newline at end of file diff --git a/packages/iis/data_stream/webserver/fields/ecs.yml b/packages/iis/data_stream/webserver/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/iis/data_stream/webserver/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/iis/data_stream/webserver/sample_event.json b/packages/iis/data_stream/webserver/sample_event.json index ae5de4f9963..5bd3d8230de 100644 --- a/packages/iis/data_stream/webserver/sample_event.json +++ b/packages/iis/data_stream/webserver/sample_event.json @@ -3,11 +3,6 @@ "service": { "type": "iis" }, - "dataset": { - "type": "metrics", - "name": "iis.webserver", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -27,9 +22,7 @@ "asp_net_application": { "requests_in_application_queue": 0, "pipeline_instance_count": 2, - "requests/sec": 0, - "requests_executing": 0, - "errors_total/sec": 0 + "requests_executing": 0 }, "network": { "total_get_requests": 52, diff --git a/packages/iis/data_stream/website/fields/ecs.yml b/packages/iis/data_stream/website/fields/ecs.yml new file mode 100644 index 00000000000..7509bf59910 --- /dev/null +++ b/packages/iis/data_stream/website/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.type + type: keyword + description: Service type diff --git a/packages/iis/data_stream/website/sample_event.json b/packages/iis/data_stream/website/sample_event.json index 86bfab602ea..27679661c71 100644 --- a/packages/iis/data_stream/website/sample_event.json +++ b/packages/iis/data_stream/website/sample_event.json @@ -32,11 +32,6 @@ "service": { "type": "iis" }, - "dataset": { - "name": "iis.website", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", diff --git a/packages/iis/docs/README.md b/packages/iis/docs/README.md index 0ac1c12f18c..06dc9238e60 100644 --- a/packages/iis/docs/README.md +++ b/packages/iis/docs/README.md @@ -23,11 +23,6 @@ An example event for `webserver` looks as following: "service": { "type": "iis" }, - "dataset": { - "type": "metrics", - "name": "iis.webserver", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -47,9 +42,7 @@ An example event for `webserver` looks as following: "asp_net_application": { "requests_in_application_queue": 0, "pipeline_instance_count": 2, - "requests/sec": 0, - "requests_executing": 0, - "errors_total/sec": 0 + "requests_executing": 0 }, "network": { "total_get_requests": 52, @@ -133,6 +126,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -199,6 +193,8 @@ The fields reported are: | iis.webserver.process.virtual_bytes | Memory virtual bytes. | float | | iis.webserver.process.worker_process_count | Number of worker processes running. | float | | iis.webserver.process.working_set | Memory working set. | float | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### website @@ -241,11 +237,6 @@ An example event for `website` looks as following: "service": { "type": "iis" }, - "dataset": { - "name": "iis.website", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -279,6 +270,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -312,6 +304,7 @@ The fields reported are: | iis.website.network.total_get_requests | The total number of GET requests. | float | | iis.website.network.total_post_requests | The total number of POST requests. | float | | iis.website.network.total_put_requests | The total number of PUT requests. | float | +| service.type | Service type | keyword | ### application_pool @@ -322,9 +315,6 @@ An example event for `application_pool` looks as following: ```$json { "@timestamp": "2020-07-08T11:41:31.048Z", - "process": { - "pid": 51224 - }, "event": { "dataset": "iis.application_pool", "module": "iis", @@ -359,11 +349,6 @@ An example event for `application_pool` looks as following: "metricset": { "period": 10000, "name": "application_pool" - }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "iis.application_pool" } } ``` @@ -391,6 +376,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -421,6 +407,8 @@ The fields reported are: | iis.application_pool.process.thread_count | The number of threats. | long | | iis.application_pool.process.virtual_bytes | Memory virtual bytes. | float | | iis.application_pool.process.working_set | Memory working set. | float | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ## Logs @@ -444,12 +432,6 @@ An example event for `access` looks as following: "version": "8.0.0" }, "temp": {}, - "log": { - "file": { - "path": "C:\\inetpub\\logs\\LogFiles\\W3SVC2\\u_ex181119.log" - }, - "offset": 261 - }, "destination": { "address": "127.0.0.1", "port": 80, @@ -462,9 +444,6 @@ An example event for `access` looks as following: "url": { "path": "/" }, - "input": { - "type": "log" - }, "iis": { "access": { "sub_status": 3, @@ -502,11 +481,6 @@ An example event for `access` looks as following: ], "outcome": "failure" }, - "dataset": { - "name": "iis.access", - "namespace": "default", - "type": "logs" - }, "user_agent": { "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "os": { @@ -550,6 +524,7 @@ The fields reported are: | destination.domain | Destination domain. | keyword | | destination.ip | | ip | | destination.port | Port of the destination. | long | +| ecs.version | ECS version | keyword | | error.message | Error message. | text | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | @@ -620,12 +595,6 @@ An example event for `error` looks as following: "ephemeral_id": "3f65b650-b6a3-4694-83b3-0c324a60809d", "version": "8.0.0" }, - "log": { - "file": { - "path": "c:\\Windows\\System32\\LogFiles\\HTTPERR\\httperr1.log" - }, - "offset": 199 - }, "destination": { "address": "::1%0", "port": 80, @@ -636,9 +605,6 @@ An example event for `error` looks as following: "port": 59827, "ip": "::1" }, - "input": { - "type": "log" - }, "iis": { "error": { "reason_phrase": "Timer_ConnectionIdle" @@ -664,11 +630,6 @@ An example event for `error` looks as following: "type": [ "connection" ] - }, - "dataset": { - "name": "iis.error", - "namespace": "default", - "type": "logs" } } ``` @@ -700,6 +661,7 @@ The fields reported are: | destination.domain | Destination domain. | keyword | | destination.ip | | ip | | destination.port | Port of the destination. | long | +| ecs.version | ECS version | keyword | | error.message | Error message | text | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | diff --git a/packages/iis/manifest.yml b/packages/iis/manifest.yml index b2265f632f4..687c933dd88 100644 --- a/packages/iis/manifest.yml +++ b/packages/iis/manifest.yml @@ -1,6 +1,6 @@ name: iis title: IIS -version: 0.3.0 +version: 0.3.1 description: IIS Integration type: integration icons: From a6e91898c7bf02e487df8a5c90c81bd2072e6da0 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:28:07 +0100 Subject: [PATCH 11/18] Fix --- packages/iis/changelog.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/iis/changelog.yml b/packages/iis/changelog.yml index 0ce9c4e983a..3982d8e1ffe 100644 --- a/packages/iis/changelog.yml +++ b/packages/iis/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top - version: "0.3.1" changes: - - description: Use correct types for `source.port` and `source.ip` - type: bugfix - link: https://github.com/elastic/integrations/pull/737 + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release From 4364c0853bf6c6fc391ca7979124c8220f597f57 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:31:04 +0100 Subject: [PATCH 12/18] Fix Kafka --- packages/kafka/changelog.yml | 5 ++++ .../kafka/data_stream/broker/fields/ecs.yml | 9 +++++++ .../data_stream/broker/sample_event.json | 5 ---- .../data_stream/consumergroup/fields/ecs.yml | 9 +++++++ .../consumergroup/sample_event.json | 5 ---- .../data_stream/partition/fields/ecs.yml | 9 +++++++ .../data_stream/partition/sample_event.json | 5 ---- packages/kafka/docs/README.md | 24 +++++++------------ packages/kafka/manifest.yml | 2 +- 9 files changed, 42 insertions(+), 31 deletions(-) create mode 100644 packages/kafka/data_stream/broker/fields/ecs.yml create mode 100644 packages/kafka/data_stream/consumergroup/fields/ecs.yml create mode 100644 packages/kafka/data_stream/partition/fields/ecs.yml diff --git a/packages/kafka/changelog.yml b/packages/kafka/changelog.yml index aa0a26f8158..792a52ffc33 100644 --- a/packages/kafka/changelog.yml +++ b/packages/kafka/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.7" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/kafka/data_stream/broker/fields/ecs.yml b/packages/kafka/data_stream/broker/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/kafka/data_stream/broker/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/kafka/data_stream/broker/sample_event.json b/packages/kafka/data_stream/broker/sample_event.json index 4f21120e727..447549c4dc4 100644 --- a/packages/kafka/data_stream/broker/sample_event.json +++ b/packages/kafka/data_stream/broker/sample_event.json @@ -25,11 +25,6 @@ "period": 10000, "name": "broker" }, - "stream": { - "type": "metrics", - "dataset": "kafka.broker", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/kafka/data_stream/consumergroup/fields/ecs.yml b/packages/kafka/data_stream/consumergroup/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/kafka/data_stream/consumergroup/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/kafka/data_stream/consumergroup/sample_event.json b/packages/kafka/data_stream/consumergroup/sample_event.json index 784c3fd747d..be757e79936 100644 --- a/packages/kafka/data_stream/consumergroup/sample_event.json +++ b/packages/kafka/data_stream/consumergroup/sample_event.json @@ -55,10 +55,5 @@ "service": { "address": "localhost:9092", "type": "kafka" - }, - "stream": { - "dataset": "kafka.consumergroup", - "namespace": "default", - "type": "metrics" } } \ No newline at end of file diff --git a/packages/kafka/data_stream/partition/fields/ecs.yml b/packages/kafka/data_stream/partition/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/kafka/data_stream/partition/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/kafka/data_stream/partition/sample_event.json b/packages/kafka/data_stream/partition/sample_event.json index 976ed7981be..e23060b8376 100644 --- a/packages/kafka/data_stream/partition/sample_event.json +++ b/packages/kafka/data_stream/partition/sample_event.json @@ -40,11 +40,6 @@ "name": "messages" } }, - "stream": { - "type": "metrics", - "dataset": "kafka.partition", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/kafka/docs/README.md b/packages/kafka/docs/README.md index 23d87a2a316..2ef3b9e79a6 100644 --- a/packages/kafka/docs/README.md +++ b/packages/kafka/docs/README.md @@ -97,11 +97,6 @@ An example event for `broker` looks as following: "period": 10000, "name": "broker" }, - "stream": { - "type": "metrics", - "dataset": "kafka.broker", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -136,6 +131,7 @@ An example event for `broker` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -180,6 +176,8 @@ An example event for `broker` looks as following: | kafka.partition.topic_id | Unique id of the partition in the topic. | keyword | | kafka.topic.error.code | Topic error code. | long | | kafka.topic.name | Topic name | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### consumergroup @@ -244,11 +242,6 @@ An example event for `consumergroup` looks as following: "service": { "address": "localhost:9092", "type": "kafka" - }, - "stream": { - "dataset": "kafka.consumergroup", - "namespace": "default", - "type": "metrics" } } ``` @@ -274,6 +267,7 @@ An example event for `consumergroup` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -309,6 +303,8 @@ An example event for `consumergroup` looks as following: | kafka.partition.topic_id | Unique id of the partition in the topic. | keyword | | kafka.topic.error.code | Topic error code. | long | | kafka.topic.name | Topic name | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### partition @@ -358,11 +354,6 @@ An example event for `partition` looks as following: "name": "messages" } }, - "stream": { - "type": "metrics", - "dataset": "kafka.partition", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -402,6 +393,7 @@ An example event for `partition` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -438,3 +430,5 @@ An example event for `partition` looks as following: | kafka.partition.topic_id | Unique id of the partition in the topic. | keyword | | kafka.topic.error.code | Topic error code. | long | | kafka.topic.name | Topic name | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/kafka/manifest.yml b/packages/kafka/manifest.yml index c9d47e41edd..c1867279aeb 100644 --- a/packages/kafka/manifest.yml +++ b/packages/kafka/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kafka title: Kafka -version: 0.3.6 +version: 0.3.7 license: basic description: Kafka Integration type: integration From 4be0529732fa935536a6d1fe34c3b96a00afb6a8 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:43:01 +0100 Subject: [PATCH 13/18] Fix mongodb --- packages/mongodb/changelog.yml | 5 + .../data_stream/collstats/fields/ecs.yml | 9 ++ .../data_stream/collstats/sample_event.json | 10 -- .../data_stream/dbstats/fields/ecs.yml | 9 ++ .../data_stream/dbstats/sample_event.json | 10 -- .../mongodb/data_stream/log/fields/ecs.yml | 9 ++ .../mongodb/data_stream/log/sample_event.json | 14 -- .../data_stream/metrics/fields/ecs.yml | 9 ++ .../data_stream/metrics/sample_event.json | 10 -- .../data_stream/replstatus/fields/ecs.yml | 9 ++ .../data_stream/replstatus/sample_event.json | 65 ++++++++-- .../mongodb/data_stream/status/fields/ecs.yml | 21 ++- .../data_stream/status/sample_event.json | 10 -- packages/mongodb/docs/README.md | 122 ++++++++++-------- packages/mongodb/manifest.yml | 2 +- 15 files changed, 181 insertions(+), 133 deletions(-) create mode 100644 packages/mongodb/data_stream/collstats/fields/ecs.yml create mode 100644 packages/mongodb/data_stream/dbstats/fields/ecs.yml create mode 100644 packages/mongodb/data_stream/metrics/fields/ecs.yml create mode 100644 packages/mongodb/data_stream/replstatus/fields/ecs.yml diff --git a/packages/mongodb/changelog.yml b/packages/mongodb/changelog.yml index d1fd9261778..cc7b0a47fef 100644 --- a/packages/mongodb/changelog.yml +++ b/packages/mongodb/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.8" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/mongodb/data_stream/collstats/fields/ecs.yml b/packages/mongodb/data_stream/collstats/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/mongodb/data_stream/collstats/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mongodb/data_stream/collstats/sample_event.json b/packages/mongodb/data_stream/collstats/sample_event.json index f1a8153f7b9..43a8d8164d5 100644 --- a/packages/mongodb/data_stream/collstats/sample_event.json +++ b/packages/mongodb/data_stream/collstats/sample_event.json @@ -8,11 +8,6 @@ "address": "localhost:27017", "type": "mongodb" }, - "stream": { - "dataset": "mongodb.collstats", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -88,11 +83,6 @@ } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.collstats" - }, "ecs": { "version": "1.5.0" } diff --git a/packages/mongodb/data_stream/dbstats/fields/ecs.yml b/packages/mongodb/data_stream/dbstats/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/mongodb/data_stream/dbstats/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mongodb/data_stream/dbstats/sample_event.json b/packages/mongodb/data_stream/dbstats/sample_event.json index a6926020b78..b5cd2866b1d 100644 --- a/packages/mongodb/data_stream/dbstats/sample_event.json +++ b/packages/mongodb/data_stream/dbstats/sample_event.json @@ -8,11 +8,6 @@ "address": "localhost:27017", "type": "mongodb" }, - "stream": { - "dataset": "mongodb.dbstats", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -48,11 +43,6 @@ } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.dbstats" - }, "ecs": { "version": "1.5.0" } diff --git a/packages/mongodb/data_stream/log/fields/ecs.yml b/packages/mongodb/data_stream/log/fields/ecs.yml index 417e524fb50..9d4a065a26e 100644 --- a/packages/mongodb/data_stream/log/fields/ecs.yml +++ b/packages/mongodb/data_stream/log/fields/ecs.yml @@ -29,3 +29,12 @@ - name: created type: date description: event.created contains the date/time when the event was first read by an agent, or by your pipeline. +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mongodb/data_stream/log/sample_event.json b/packages/mongodb/data_stream/log/sample_event.json index 645b270356e..3db864ab2ff 100644 --- a/packages/mongodb/data_stream/log/sample_event.json +++ b/packages/mongodb/data_stream/log/sample_event.json @@ -13,22 +13,13 @@ "file": { "path": "/usr/local/var/log/mongodb/mongo.log" }, - "offset": 584519, "level": "I" }, "message": "end connection 127.0.0.1:60764 (1 connection now open)", - "input": { - "type": "log" - }, "@timestamp": "2020-06-29T21:17:11.459Z", "ecs": { "version": "1.5.0" }, - "stream": { - "namespace": "default", - "type": "logs", - "dataset": "mongodb.log" - }, "event": { "created": "2020-06-29T21:17:12.442Z", "kind": "event", @@ -39,11 +30,6 @@ "info" ] }, - "dataset": { - "namespace": "default", - "name": "mongodb.log", - "type": "logs" - }, "mongodb": { "log": { "component": "NETWORK", diff --git a/packages/mongodb/data_stream/metrics/fields/ecs.yml b/packages/mongodb/data_stream/metrics/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/mongodb/data_stream/metrics/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mongodb/data_stream/metrics/sample_event.json b/packages/mongodb/data_stream/metrics/sample_event.json index 8a628a45e8c..cf7fdda0851 100644 --- a/packages/mongodb/data_stream/metrics/sample_event.json +++ b/packages/mongodb/data_stream/metrics/sample_event.json @@ -219,11 +219,6 @@ "period": 10000, "name": "metrics" }, - "stream": { - "type": "metrics", - "dataset": "mongodb.metrics", - "namespace": "default" - }, "agent": { "name": "KaiyanMacBookPro", "type": "metricbeat", @@ -240,11 +235,6 @@ "module": "mongodb", "duration": 3039885 }, - "dataset": { - "type": "metrics", - "name": "mongodb.metrics", - "namespace": "default" - }, "ecs": { "version": "1.5.0" } diff --git a/packages/mongodb/data_stream/replstatus/fields/ecs.yml b/packages/mongodb/data_stream/replstatus/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/mongodb/data_stream/replstatus/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mongodb/data_stream/replstatus/sample_event.json b/packages/mongodb/data_stream/replstatus/sample_event.json index 04d5a09f750..176e4c50378 100644 --- a/packages/mongodb/data_stream/replstatus/sample_event.json +++ b/packages/mongodb/data_stream/replstatus/sample_event.json @@ -4,18 +4,59 @@ "address": "localhost:27017", "type": "mongodb" }, - "error": { - "message": "error getting replication info: collection oplog.rs was not found" - }, - "dataset": { - "name": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" + "mongodb": { + "replstatus": { + "members": { + "arbiter": { + "count": 0 + }, + "down": { + "count": 0 + }, + "primary": { + "host": "22b4e1fb8197:27017", + "optime": 1550700559 + }, + "recovering": { + "count": 0 + }, + "rollback": { + "count": 0 + }, + "secondary": { + "count": 0 + }, + "startup2": { + "count": 0 + }, + "unhealthy": { + "count": 0 + }, + "unknown": { + "count": 0 + } + }, + "oplog": { + "first": { + "timestamp": 1550700557 + }, + "last": { + "timestamp": 1550700559 + }, + "size": { + "allocated": 40572728934, + "used": 180 + }, + "window": 2 + }, + "optimes": { + "applied": 1550700559, + "durable": 1550700559, + "last_committed": 1550700559 + }, + "server_date": "2019-02-20T23:09:23.733+01:00", + "set_name": "beats" + } }, "ecs": { "version": "1.5.0" diff --git a/packages/mongodb/data_stream/status/fields/ecs.yml b/packages/mongodb/data_stream/status/fields/ecs.yml index 7445bc5828c..221550b9b87 100644 --- a/packages/mongodb/data_stream/status/fields/ecs.yml +++ b/packages/mongodb/data_stream/status/fields/ecs.yml @@ -15,15 +15,12 @@ type: text norms: false default_field: false -- name: service - title: Service - group: 2 - type: group - fields: - - name: version - level: core - type: keyword - description: |- - Version of the service the data was collected from. - This allows to look at a data set only for a specific version of a service. - ignore_above: 1024 +- name: ecs.version + type: keyword + description: ECS version +- name: service.type + type: keyword + description: Service type +- name: service.version + type: keyword + description: Service version diff --git a/packages/mongodb/data_stream/status/sample_event.json b/packages/mongodb/data_stream/status/sample_event.json index a9ba62435bf..cb2d6153293 100644 --- a/packages/mongodb/data_stream/status/sample_event.json +++ b/packages/mongodb/data_stream/status/sample_event.json @@ -1,15 +1,5 @@ { "@timestamp": "2020-06-29T21:20:01.455Z", - "dataset": { - "type": "metrics", - "name": "mongodb.status", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "mongodb.status", - "namespace": "default" - }, "agent": { "version": "8.0.0", "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", diff --git a/packages/mongodb/docs/README.md b/packages/mongodb/docs/README.md index 6d2c16b987d..fa035fb950f 100644 --- a/packages/mongodb/docs/README.md +++ b/packages/mongodb/docs/README.md @@ -63,6 +63,7 @@ The `log` dataset collects the MongoDB logs. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. | date | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | @@ -85,6 +86,8 @@ The `log` dataset collects the MongoDB logs. | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | | mongodb.log.component | Functional categorization of message | keyword | | mongodb.log.context | Context of message | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ## Metrics @@ -113,11 +116,6 @@ An example event for `collstats` looks as following: "address": "localhost:27017", "type": "mongodb" }, - "stream": { - "dataset": "mongodb.collstats", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -193,11 +191,6 @@ An example event for `collstats` looks as following: } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.collstats" - }, "ecs": { "version": "1.5.0" } @@ -227,6 +220,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -265,6 +259,7 @@ The fields reported are: | mongodb.collstats.update.count | Number of document update events. | long | | mongodb.collstats.update.time.us | Time updating documents in microseconds. | long | | service.address | Address of the machine where the service is running. | keyword | +| service.type | Service type | keyword | ### dbstats @@ -292,11 +287,6 @@ An example event for `dbstats` looks as following: "address": "localhost:27017", "type": "mongodb" }, - "stream": { - "dataset": "mongodb.dbstats", - "namespace": "default", - "type": "metrics" - }, "agent": { "type": "metricbeat", "version": "8.0.0", @@ -332,11 +322,6 @@ An example event for `dbstats` looks as following: } } }, - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "mongodb.dbstats" - }, "ecs": { "version": "1.5.0" } @@ -366,6 +351,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -398,6 +384,7 @@ The fields reported are: | mongodb.dbstats.objects | | long | | mongodb.dbstats.storage_size.bytes | | long | | service.address | Address of the machine where the service is running. | keyword | +| service.type | Service type | keyword | ### metrics @@ -631,11 +618,6 @@ An example event for `metrics` looks as following: "period": 10000, "name": "metrics" }, - "stream": { - "type": "metrics", - "dataset": "mongodb.metrics", - "namespace": "default" - }, "agent": { "name": "KaiyanMacBookPro", "type": "metricbeat", @@ -652,11 +634,6 @@ An example event for `metrics` looks as following: "module": "mongodb", "duration": 3039885 }, - "dataset": { - "type": "metrics", - "name": "mongodb.metrics", - "namespace": "default" - }, "ecs": { "version": "1.5.0" } @@ -686,6 +663,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -821,7 +799,8 @@ The fields reported are: | mongodb.metrics.storage.free_list.search.scanned | The number of available record allocations mongod has searched. | long | | mongodb.metrics.ttl.deleted_documents.count | The total number of documents deleted from collections with a ttl index. | long | | mongodb.metrics.ttl.passes.count | The number of times the background process removes documents from collections with a ttl index. | long | -| service.address | Address of the machine where the service is running. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### replstatus @@ -841,18 +820,59 @@ An example event for `replstatus` looks as following: "address": "localhost:27017", "type": "mongodb" }, - "error": { - "message": "error getting replication info: collection oplog.rs was not found" - }, - "dataset": { - "name": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" - }, - "stream": { - "dataset": "mongodb.replstatus", - "namespace": "default", - "type": "metrics" + "mongodb": { + "replstatus": { + "members": { + "arbiter": { + "count": 0 + }, + "down": { + "count": 0 + }, + "primary": { + "host": "22b4e1fb8197:27017", + "optime": 1550700559 + }, + "recovering": { + "count": 0 + }, + "rollback": { + "count": 0 + }, + "secondary": { + "count": 0 + }, + "startup2": { + "count": 0 + }, + "unhealthy": { + "count": 0 + }, + "unknown": { + "count": 0 + } + }, + "oplog": { + "first": { + "timestamp": 1550700557 + }, + "last": { + "timestamp": 1550700559 + }, + "size": { + "allocated": 40572728934, + "used": 180 + }, + "window": 2 + }, + "optimes": { + "applied": 1550700559, + "durable": 1550700559, + "last_committed": 1550700559 + }, + "server_date": "2019-02-20T23:09:23.733+01:00", + "set_name": "beats" + } }, "ecs": { "version": "1.5.0" @@ -899,6 +919,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -949,6 +970,7 @@ The fields reported are: | mongodb.replstatus.server_date | Reflects the current time according to the server that processed the replSetGetStatus command. | date | | mongodb.replstatus.set_name | The name of the replica set. | keyword | | service.address | Address of the machine where the service is running. | keyword | +| service.type | Service type | keyword | ### status @@ -963,16 +985,6 @@ An example event for `status` looks as following: ```$json { "@timestamp": "2020-06-29T21:20:01.455Z", - "dataset": { - "type": "metrics", - "name": "mongodb.status", - "namespace": "default" - }, - "stream": { - "type": "metrics", - "dataset": "mongodb.status", - "namespace": "default" - }, "agent": { "version": "8.0.0", "ephemeral_id": "9f6fc260-82b5-4630-95d8-df64f1379b55", @@ -1200,6 +1212,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1382,5 +1395,6 @@ The fields reported are: | mongodb.status.write_backs_queued | True when there are operations from a mongos instance queued for retrying. | boolean | | process.name | Process name. Sometimes called program name or similar. | keyword | | service.address | Address of the machine where the service is running. | keyword | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | keyword | +| service.type | Service type | keyword | +| service.version | Service version | keyword | diff --git a/packages/mongodb/manifest.yml b/packages/mongodb/manifest.yml index f29dc3c5168..c7e630bfc5b 100644 --- a/packages/mongodb/manifest.yml +++ b/packages/mongodb/manifest.yml @@ -1,6 +1,6 @@ name: mongodb title: MongoDB -version: 0.2.7 +version: 0.2.8 description: MongoDB Integration type: integration categories: From b553fb3374701841558bbbfdb994ec49458d03d2 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:46:39 +0100 Subject: [PATCH 14/18] Fix MySQL --- packages/mysql/changelog.yml | 5 +++++ .../data_stream/galera_status/fields/ecs.yml | 9 +++++++++ .../galera_status/sample_event.json | 7 ------- .../mysql/data_stream/status/fields/ecs.yml | 9 +++++++++ .../data_stream/status/sample_event.json | 7 ------- packages/mysql/docs/README.md | 20 ++++++------------- packages/mysql/manifest.yml | 2 +- 7 files changed, 30 insertions(+), 29 deletions(-) create mode 100644 packages/mysql/data_stream/galera_status/fields/ecs.yml create mode 100644 packages/mysql/data_stream/status/fields/ecs.yml diff --git a/packages/mysql/changelog.yml b/packages/mysql/changelog.yml index 5431c5f99c1..ab6da8ed0d0 100644 --- a/packages/mysql/changelog.yml +++ b/packages/mysql/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.7" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/mysql/data_stream/galera_status/fields/ecs.yml b/packages/mysql/data_stream/galera_status/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/mysql/data_stream/galera_status/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mysql/data_stream/galera_status/sample_event.json b/packages/mysql/data_stream/galera_status/sample_event.json index 70caa1bc2cb..7747ae79331 100644 --- a/packages/mysql/data_stream/galera_status/sample_event.json +++ b/packages/mysql/data_stream/galera_status/sample_event.json @@ -67,13 +67,6 @@ } } }, - "fields": { - "stream": { - "type": "metrics", - "dataset": "mysql.galera_status", - "namespace": "default" - } - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/mysql/data_stream/status/fields/ecs.yml b/packages/mysql/data_stream/status/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/mysql/data_stream/status/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/mysql/data_stream/status/sample_event.json b/packages/mysql/data_stream/status/sample_event.json index 0fba321be65..450e6529476 100644 --- a/packages/mysql/data_stream/status/sample_event.json +++ b/packages/mysql/data_stream/status/sample_event.json @@ -110,13 +110,6 @@ "name": "status", "period": 10000 }, - "fields": { - "stream": { - "type": "metrics", - "dataset": "mysql.status", - "namespace": "default" - } - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/mysql/docs/README.md b/packages/mysql/docs/README.md index 711c219924b..94f69510fed 100644 --- a/packages/mysql/docs/README.md +++ b/packages/mysql/docs/README.md @@ -233,13 +233,6 @@ An example event for `galera_status` looks as following: } } }, - "fields": { - "stream": { - "type": "metrics", - "dataset": "mysql.galera_status", - "namespace": "default" - } - }, "ecs": { "version": "1.5.0" }, @@ -287,6 +280,7 @@ An example event for `galera_status` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -344,6 +338,8 @@ An example event for `galera_status` looks as following: | mysql.galera_status.repl.keys | Total number of keys replicated. | long | | mysql.galera_status.repl.keys_bytes | Total size of keys replicated. | long | | mysql.galera_status.repl.other_bytes | Total size of other bits replicated. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### status @@ -465,13 +461,6 @@ An example event for `status` looks as following: "name": "status", "period": 10000 }, - "fields": { - "stream": { - "type": "metrics", - "dataset": "mysql.status", - "namespace": "default" - } - }, "ecs": { "version": "1.5.0" }, @@ -510,6 +499,7 @@ An example event for `status` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -591,4 +581,6 @@ An example event for `status` looks as following: | mysql.status.threads.connected | The number of connected threads. | long | | mysql.status.threads.created | The number of created threads. | long | | mysql.status.threads.running | The number of running threads. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/mysql/manifest.yml b/packages/mysql/manifest.yml index 35c0bb6f6f8..aa388d4d25f 100644 --- a/packages/mysql/manifest.yml +++ b/packages/mysql/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: mysql title: MySQL -version: 0.3.6 +version: 0.3.7 license: basic description: MySQL Integration type: integration From 383dba925c9a16255a88b0482c921c7fc2824fc1 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:52:14 +0100 Subject: [PATCH 15/18] Fix Postgresql --- packages/postgresql/changelog.yml | 5 +++++ .../postgresql/data_stream/activity/fields/ecs.yml | 6 ++++++ .../postgresql/data_stream/bgwriter/fields/ecs.yml | 6 ++++++ .../postgresql/data_stream/database/fields/ecs.yml | 6 ++++++ .../data_stream/database/sample_event.json | 4 ---- .../data_stream/statement/fields/ecs.yml | 6 ++++++ .../data_stream/statement/sample_event.json | 2 +- packages/postgresql/docs/README.md | 14 +++++++++----- packages/postgresql/manifest.yml | 2 +- 9 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 packages/postgresql/data_stream/activity/fields/ecs.yml create mode 100644 packages/postgresql/data_stream/bgwriter/fields/ecs.yml create mode 100644 packages/postgresql/data_stream/database/fields/ecs.yml create mode 100644 packages/postgresql/data_stream/statement/fields/ecs.yml diff --git a/packages/postgresql/changelog.yml b/packages/postgresql/changelog.yml index 7139cd139d5..154ac06062c 100644 --- a/packages/postgresql/changelog.yml +++ b/packages/postgresql/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.6" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/postgresql/data_stream/activity/fields/ecs.yml b/packages/postgresql/data_stream/activity/fields/ecs.yml new file mode 100644 index 00000000000..50e1946dc41 --- /dev/null +++ b/packages/postgresql/data_stream/activity/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/postgresql/data_stream/bgwriter/fields/ecs.yml b/packages/postgresql/data_stream/bgwriter/fields/ecs.yml new file mode 100644 index 00000000000..50e1946dc41 --- /dev/null +++ b/packages/postgresql/data_stream/bgwriter/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/postgresql/data_stream/database/fields/ecs.yml b/packages/postgresql/data_stream/database/fields/ecs.yml new file mode 100644 index 00000000000..50e1946dc41 --- /dev/null +++ b/packages/postgresql/data_stream/database/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/postgresql/data_stream/database/sample_event.json b/packages/postgresql/data_stream/database/sample_event.json index 1c6b7b66c99..81d32118144 100644 --- a/packages/postgresql/data_stream/database/sample_event.json +++ b/packages/postgresql/data_stream/database/sample_event.json @@ -1,9 +1,5 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "metricset": { "host": "postgresql:5432", "module": "postgresql", diff --git a/packages/postgresql/data_stream/statement/fields/ecs.yml b/packages/postgresql/data_stream/statement/fields/ecs.yml new file mode 100644 index 00000000000..50e1946dc41 --- /dev/null +++ b/packages/postgresql/data_stream/statement/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/postgresql/data_stream/statement/sample_event.json b/packages/postgresql/data_stream/statement/sample_event.json index a10d067e7e6..f94d1f92665 100644 --- a/packages/postgresql/data_stream/statement/sample_event.json +++ b/packages/postgresql/data_stream/statement/sample_event.json @@ -19,7 +19,7 @@ }, "query": { "calls": 2, - "id": "1592910677", + "id": 159291067, "memory": { "local": { "dirtied": 0, diff --git a/packages/postgresql/docs/README.md b/packages/postgresql/docs/README.md index aedc5941db5..6b9f34fc897 100644 --- a/packages/postgresql/docs/README.md +++ b/packages/postgresql/docs/README.md @@ -178,6 +178,8 @@ An example event for `activity` looks as following: | postgresql.activity.user.id | OID of the user logged into this backend. | long | | postgresql.activity.user.name | Name of the user logged into this backend. | keyword | | postgresql.activity.waiting | True if this backend is currently waiting on a lock. | boolean | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### bgwriter @@ -281,6 +283,8 @@ An example event for `bgwriter` looks as following: | postgresql.bgwriter.checkpoints.times.sync.ms | Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk, in milliseconds. | float | | postgresql.bgwriter.checkpoints.times.write.ms | Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds. | float | | postgresql.bgwriter.stats_reset | Time at which these statistics were last reset. | date | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### database @@ -292,10 +296,6 @@ An example event for `database` looks as following: ```$json { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "metricset": { "host": "postgresql:5432", "module": "postgresql", @@ -397,6 +397,8 @@ An example event for `database` looks as following: | postgresql.database.temporary.files | Number of temporary files created by queries in this database. All temporary files are counted, regardless of why the temporary file was created (e.g., sorting or hashing), and regardless of the log_temp_files setting. | long | | postgresql.database.transactions.commit | Number of transactions in this database that have been committed. | long | | postgresql.database.transactions.rollback | Number of transactions in this database that have been rolled back. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### statement @@ -427,7 +429,7 @@ An example event for `statement` looks as following: }, "query": { "calls": 2, - "id": "1592910677", + "id": 159291067, "memory": { "local": { "dirtied": 0, @@ -536,3 +538,5 @@ An example event for `statement` looks as following: | postgresql.statement.query.time.stddev.ms | Population standard deviation of time spent running query, in milliseconds. | long | | postgresql.statement.query.time.total.ms | Total number of milliseconds spent running query. | float | | postgresql.statement.user.id | OID of the user logged into the backend that ran the query. | long | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/postgresql/manifest.yml b/packages/postgresql/manifest.yml index 7f7c7e43d03..f9d61d7f8a8 100644 --- a/packages/postgresql/manifest.yml +++ b/packages/postgresql/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: postgresql title: PostgreSQL -version: 0.2.5 +version: 0.2.6 license: basic description: PostgreSQL Integration type: integration From 194b70530eec06edb71a23fa546500afa3f3deb6 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 11:55:29 +0100 Subject: [PATCH 16/18] Fix Prometheus --- packages/prometheus/changelog.yml | 5 ++++ .../data_stream/collector/fields/ecs.yml | 9 ++++++ .../data_stream/collector/sample_event.json | 5 ---- .../data_stream/query/fields/ecs.yml | 9 ++++++ .../data_stream/query/sample_event.json | 10 ------- .../data_stream/remote_write/fields/ecs.yml | 9 ++++++ .../remote_write/sample_event.json | 5 ---- packages/prometheus/docs/README.md | 29 ++++++------------- packages/prometheus/manifest.yml | 2 +- 9 files changed, 42 insertions(+), 41 deletions(-) create mode 100644 packages/prometheus/data_stream/collector/fields/ecs.yml create mode 100644 packages/prometheus/data_stream/query/fields/ecs.yml create mode 100644 packages/prometheus/data_stream/remote_write/fields/ecs.yml diff --git a/packages/prometheus/changelog.yml b/packages/prometheus/changelog.yml index a7df2f9d799..0ccac83b67c 100644 --- a/packages/prometheus/changelog.yml +++ b/packages/prometheus/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.4" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.3.3" changes: - description: Change kibana.version constraint to be more conservative. diff --git a/packages/prometheus/data_stream/collector/fields/ecs.yml b/packages/prometheus/data_stream/collector/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/prometheus/data_stream/collector/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/prometheus/data_stream/collector/sample_event.json b/packages/prometheus/data_stream/collector/sample_event.json index 152380e0419..8a4b3fec4a5 100644 --- a/packages/prometheus/data_stream/collector/sample_event.json +++ b/packages/prometheus/data_stream/collector/sample_event.json @@ -25,11 +25,6 @@ "instance": "localhost:9090" } }, - "dataset": { - "type": "metrics", - "name": "prometheus.collector", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/prometheus/data_stream/query/fields/ecs.yml b/packages/prometheus/data_stream/query/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/prometheus/data_stream/query/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/prometheus/data_stream/query/sample_event.json b/packages/prometheus/data_stream/query/sample_event.json index 7a880a964d8..3224615a989 100644 --- a/packages/prometheus/data_stream/query/sample_event.json +++ b/packages/prometheus/data_stream/query/sample_event.json @@ -17,16 +17,6 @@ "name": "query", "period": 10000 }, - "dataset": { - "type": "metrics", - "name": "prometheus.query", - "namespace": "default" - }, - "stream": { - "dataset": "prometheus.query", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, diff --git a/packages/prometheus/data_stream/remote_write/fields/ecs.yml b/packages/prometheus/data_stream/remote_write/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/prometheus/data_stream/remote_write/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/prometheus/data_stream/remote_write/sample_event.json b/packages/prometheus/data_stream/remote_write/sample_event.json index 3458d4ba3bd..7f720e32892 100644 --- a/packages/prometheus/data_stream/remote_write/sample_event.json +++ b/packages/prometheus/data_stream/remote_write/sample_event.json @@ -31,10 +31,5 @@ "event": { "dataset": "prometheus.remote_write", "module": "prometheus" - }, - "dataset": { - "type": "metrics", - "name": "prometheus.remote_write", - "namespace": "default" } } \ No newline at end of file diff --git a/packages/prometheus/docs/README.md b/packages/prometheus/docs/README.md index e8756261b10..7a6dcf0972f 100644 --- a/packages/prometheus/docs/README.md +++ b/packages/prometheus/docs/README.md @@ -156,11 +156,6 @@ An example event for `collector` looks as following: "instance": "localhost:9090" } }, - "dataset": { - "type": "metrics", - "name": "prometheus.collector", - "namespace": "default" - }, "ecs": { "version": "1.5.0" }, @@ -191,6 +186,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -213,6 +209,8 @@ The fields reported are: | prometheus.*.value | Prometheus gauge metric | object | | prometheus.labels.* | Prometheus metric labels | object | | prometheus.metrics.* | Prometheus metric | object | +| service.address | Service address | keyword | +| service.type | Service type | keyword | @@ -310,11 +308,6 @@ An example event for `remote_write` looks as following: "event": { "dataset": "prometheus.remote_write", "module": "prometheus" - }, - "dataset": { - "type": "metrics", - "name": "prometheus.remote_write", - "namespace": "default" } } ``` @@ -342,6 +335,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -364,6 +358,8 @@ The fields reported are: | prometheus.*.value | Prometheus gauge metric | object | | prometheus.labels.* | Prometheus metric labels | object | | prometheus.metrics.* | Prometheus metric | object | +| service.address | Service address | keyword | +| service.type | Service type | keyword | #### Histograms and types [x-pack] @@ -506,16 +502,6 @@ An example event for `query` looks as following: "name": "query", "period": 10000 }, - "dataset": { - "type": "metrics", - "name": "prometheus.query", - "namespace": "default" - }, - "stream": { - "dataset": "prometheus.query", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, @@ -555,6 +541,7 @@ The fields reported are: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -573,3 +560,5 @@ The fields reported are: | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | prometheus.labels.* | Prometheus metric labels | object | | prometheus.query.* | Prometheus value resulted from PromQL | object | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/prometheus/manifest.yml b/packages/prometheus/manifest.yml index 7e8644f9ac6..3a99ab24463 100644 --- a/packages/prometheus/manifest.yml +++ b/packages/prometheus/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: prometheus title: Prometheus -version: 0.3.3 +version: 0.3.4 license: basic description: Prometheus Integration type: integration From 712640fae1a46a430fd52ad8f102d2e4d16c1c0a Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 12:08:11 +0100 Subject: [PATCH 17/18] Fix RabbitMQ --- packages/rabbitmq/changelog.yml | 5 + .../data_stream/connection/fields/ecs.yml | 9 + .../data_stream/connection/sample_event.json | 10 - .../data_stream/exchange/fields/ecs.yml | 9 + .../data_stream/exchange/sample_event.json | 11 -- .../rabbitmq/data_stream/node/fields/ecs.yml | 9 + .../data_stream/node/sample_event.json | 136 +++++++++++-- .../rabbitmq/data_stream/queue/fields/ecs.yml | 9 + .../data_stream/queue/sample_event.json | 13 -- packages/rabbitmq/docs/README.md | 182 ++++++++++++------ packages/rabbitmq/manifest.yml | 2 +- 11 files changed, 286 insertions(+), 109 deletions(-) create mode 100644 packages/rabbitmq/data_stream/node/fields/ecs.yml create mode 100644 packages/rabbitmq/data_stream/queue/fields/ecs.yml diff --git a/packages/rabbitmq/changelog.yml b/packages/rabbitmq/changelog.yml index ade11b08313..76f527a9663 100644 --- a/packages/rabbitmq/changelog.yml +++ b/packages/rabbitmq/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.7" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 - version: "0.1.0" changes: - description: initial release diff --git a/packages/rabbitmq/data_stream/connection/fields/ecs.yml b/packages/rabbitmq/data_stream/connection/fields/ecs.yml index bb9360c01cb..abf28bcec6d 100644 --- a/packages/rabbitmq/data_stream/connection/fields/ecs.yml +++ b/packages/rabbitmq/data_stream/connection/fields/ecs.yml @@ -13,3 +13,12 @@ type: text norms: false default_field: false +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/rabbitmq/data_stream/connection/sample_event.json b/packages/rabbitmq/data_stream/connection/sample_event.json index 968001e2f2e..d9c2041e59d 100644 --- a/packages/rabbitmq/data_stream/connection/sample_event.json +++ b/packages/rabbitmq/data_stream/connection/sample_event.json @@ -1,10 +1,5 @@ { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "rabbitmq.connection", - "namespace": "default", - "type": "metrics" - }, "rabbitmq": { "vhost": "/", "connection": { @@ -39,11 +34,6 @@ "dataset": "rabbitmq.connection", "module": "rabbitmq" }, - "stream": { - "dataset": "rabbitmq.connection", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "connection", "period": 10000 diff --git a/packages/rabbitmq/data_stream/exchange/fields/ecs.yml b/packages/rabbitmq/data_stream/exchange/fields/ecs.yml index bb9360c01cb..abf28bcec6d 100644 --- a/packages/rabbitmq/data_stream/exchange/fields/ecs.yml +++ b/packages/rabbitmq/data_stream/exchange/fields/ecs.yml @@ -13,3 +13,12 @@ type: text norms: false default_field: false +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/rabbitmq/data_stream/exchange/sample_event.json b/packages/rabbitmq/data_stream/exchange/sample_event.json index a2a56fb6804..3aefa46d1bd 100644 --- a/packages/rabbitmq/data_stream/exchange/sample_event.json +++ b/packages/rabbitmq/data_stream/exchange/sample_event.json @@ -1,15 +1,9 @@ { "@timestamp": "2020-06-25T10:04:20.944Z", - "dataset": { - "name": "rabbitmq.exchange", - "namespace": "default", - "type": "metrics" - }, "rabbitmq": { "vhost": "/", "exchange": { "arguments": {}, - "type": "direct", "durable": true, "auto_delete": false, "name": "", @@ -21,11 +15,6 @@ "dataset": "rabbitmq.exchange", "module": "rabbitmq" }, - "stream": { - "dataset": "rabbitmq.exchange", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "exchange", "period": 10000 diff --git a/packages/rabbitmq/data_stream/node/fields/ecs.yml b/packages/rabbitmq/data_stream/node/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/rabbitmq/data_stream/node/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/rabbitmq/data_stream/node/sample_event.json b/packages/rabbitmq/data_stream/node/sample_event.json index 03d4efe23c9..a8c35d27a15 100644 --- a/packages/rabbitmq/data_stream/node/sample_event.json +++ b/packages/rabbitmq/data_stream/node/sample_event.json @@ -1,36 +1,132 @@ { "@timestamp": "2020-06-25T10:04:20.944Z", - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "rabbitmq.exchange" - }, "rabbitmq": { "vhost": "/", - "exchange": { - "type": "fanout", - "durable": true, - "auto_delete": false, - "internal": false, - "name": "amq.fanout", - "arguments": {} + "node": { + "disk": { + "free": { + "bytes": 485213712384, + "limit": { + "bytes": 50000000 + } + } + }, + "fd": { + "total": 1048576, + "used": 54 + }, + "gc": { + "num": { + "count": 5724 + }, + "reclaimed": { + "bytes": 294021640 + } + }, + "io": { + "file_handle": { + "open_attempt": { + "avg": { + "ms": 0 + }, + "count": 10 + } + }, + "read": { + "avg": { + "ms": 0 + }, + "bytes": 1, + "count": 1 + }, + "reopen": { + "count": 1 + }, + "seek": { + "avg": { + "ms": 0 + }, + "count": 0 + }, + "sync": { + "avg": { + "ms": 0 + }, + "count": 0 + }, + "write": { + "avg": { + "ms": 0 + }, + "bytes": 0, + "count": 0 + } + }, + "mem": { + "limit": { + "bytes": 13340778496 + }, + "used": { + "bytes": 71448312 + } + }, + "mnesia": { + "disk": { + "tx": { + "count": 0 + } + }, + "ram": { + "tx": { + "count": 43 + } + } + }, + "msg": { + "store_read": { + "count": 0 + }, + "store_write": { + "count": 0 + } + }, + "name": "rabbit@my-rabbit", + "proc": { + "total": 1048576, + "used": 234 + }, + "processors": 12, + "queue": { + "index": { + "journal_write": { + "count": 0 + }, + "read": { + "count": 0 + }, + "write": { + "count": 0 + } + } + }, + "run": { + "queue": 0 + }, + "socket": { + "total": 943626, + "used": 0 + }, + "type": "disc", + "uptime": 155275 } }, "metricset": { "name": "exchange", "period": 10000 }, - "user": { - "name": "rmq-internal" - }, "ecs": { "version": "1.5.0" }, - "stream": { - "type": "metrics", - "dataset": "rabbitmq.exchange", - "namespace": "default" - }, "service": { "address": "localhost:15672", "type": "rabbitmq" diff --git a/packages/rabbitmq/data_stream/queue/fields/ecs.yml b/packages/rabbitmq/data_stream/queue/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/rabbitmq/data_stream/queue/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/rabbitmq/data_stream/queue/sample_event.json b/packages/rabbitmq/data_stream/queue/sample_event.json index 133551aef7d..cb377435983 100644 --- a/packages/rabbitmq/data_stream/queue/sample_event.json +++ b/packages/rabbitmq/data_stream/queue/sample_event.json @@ -1,14 +1,6 @@ { "@timestamp": "2020-06-25T10:15:10.955Z", - "dataset": { - "type": "metrics", - "name": "rabbitmq.queue", - "namespace": "default" - }, "rabbitmq": { - "node": { - "name": "rabbit@047b9c4733f5" - }, "queue": { "auto_delete": false, "state": "running", @@ -66,11 +58,6 @@ "type": "rabbitmq", "address": "localhost:15672" }, - "stream": { - "dataset": "rabbitmq.queue", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" } diff --git a/packages/rabbitmq/docs/README.md b/packages/rabbitmq/docs/README.md index 93912cb78ec..a23ba7dc592 100644 --- a/packages/rabbitmq/docs/README.md +++ b/packages/rabbitmq/docs/README.md @@ -69,11 +69,6 @@ An example event for `connection` looks as following: ```$json { "@timestamp": "2020-06-25T10:16:10.138Z", - "dataset": { - "name": "rabbitmq.connection", - "namespace": "default", - "type": "metrics" - }, "rabbitmq": { "vhost": "/", "connection": { @@ -108,11 +103,6 @@ An example event for `connection` looks as following: "dataset": "rabbitmq.connection", "module": "rabbitmq" }, - "stream": { - "dataset": "rabbitmq.connection", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "connection", "period": 10000 @@ -148,6 +138,7 @@ An example event for `connection` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -181,6 +172,8 @@ An example event for `connection` looks as following: | rabbitmq.connection.state | Connection state. | keyword | | rabbitmq.connection.type | Type of the connection. | keyword | | rabbitmq.vhost | Virtual host name with non-ASCII characters escaped as in C. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | | user.name | Short name or login of the user. | keyword | @@ -191,16 +184,10 @@ An example event for `exchange` looks as following: ```$json { "@timestamp": "2020-06-25T10:04:20.944Z", - "dataset": { - "name": "rabbitmq.exchange", - "namespace": "default", - "type": "metrics" - }, "rabbitmq": { "vhost": "/", "exchange": { "arguments": {}, - "type": "direct", "durable": true, "auto_delete": false, "name": "", @@ -212,11 +199,6 @@ An example event for `exchange` looks as following: "dataset": "rabbitmq.exchange", "module": "rabbitmq" }, - "stream": { - "dataset": "rabbitmq.exchange", - "namespace": "default", - "type": "metrics" - }, "metricset": { "name": "exchange", "period": 10000 @@ -255,6 +237,7 @@ An example event for `exchange` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -280,6 +263,8 @@ An example event for `exchange` looks as following: | rabbitmq.exchange.messages.publish_out.details.rate | How much the exchange publish-out count has changed per second in the most recent sampling interval. | float | | rabbitmq.exchange.name | The name of the queue with non-ASCII characters escaped as in C. | keyword | | rabbitmq.vhost | Virtual host name with non-ASCII characters escaped as in C. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | | user.name | Short name or login of the user. | keyword | @@ -297,37 +282,133 @@ An example event for `node` looks as following: ```$json { "@timestamp": "2020-06-25T10:04:20.944Z", - "dataset": { - "namespace": "default", - "type": "metrics", - "name": "rabbitmq.exchange" - }, "rabbitmq": { "vhost": "/", - "exchange": { - "type": "fanout", - "durable": true, - "auto_delete": false, - "internal": false, - "name": "amq.fanout", - "arguments": {} + "node": { + "disk": { + "free": { + "bytes": 485213712384, + "limit": { + "bytes": 50000000 + } + } + }, + "fd": { + "total": 1048576, + "used": 54 + }, + "gc": { + "num": { + "count": 5724 + }, + "reclaimed": { + "bytes": 294021640 + } + }, + "io": { + "file_handle": { + "open_attempt": { + "avg": { + "ms": 0 + }, + "count": 10 + } + }, + "read": { + "avg": { + "ms": 0 + }, + "bytes": 1, + "count": 1 + }, + "reopen": { + "count": 1 + }, + "seek": { + "avg": { + "ms": 0 + }, + "count": 0 + }, + "sync": { + "avg": { + "ms": 0 + }, + "count": 0 + }, + "write": { + "avg": { + "ms": 0 + }, + "bytes": 0, + "count": 0 + } + }, + "mem": { + "limit": { + "bytes": 13340778496 + }, + "used": { + "bytes": 71448312 + } + }, + "mnesia": { + "disk": { + "tx": { + "count": 0 + } + }, + "ram": { + "tx": { + "count": 43 + } + } + }, + "msg": { + "store_read": { + "count": 0 + }, + "store_write": { + "count": 0 + } + }, + "name": "rabbit@my-rabbit", + "proc": { + "total": 1048576, + "used": 234 + }, + "processors": 12, + "queue": { + "index": { + "journal_write": { + "count": 0 + }, + "read": { + "count": 0 + }, + "write": { + "count": 0 + } + } + }, + "run": { + "queue": 0 + }, + "socket": { + "total": 943626, + "used": 0 + }, + "type": "disc", + "uptime": 155275 } }, "metricset": { "name": "exchange", "period": 10000 }, - "user": { - "name": "rmq-internal" - }, "ecs": { "version": "1.5.0" }, - "stream": { - "type": "metrics", - "dataset": "rabbitmq.exchange", - "namespace": "default" - }, "service": { "address": "localhost:15672", "type": "rabbitmq" @@ -361,6 +442,7 @@ An example event for `node` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -415,6 +497,8 @@ An example event for `node` looks as following: | rabbitmq.node.type | Node type. | keyword | | rabbitmq.node.uptime | Node uptime. | long | | rabbitmq.vhost | Virtual host name with non-ASCII characters escaped as in C. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | ### Queue Metrics @@ -424,15 +508,7 @@ An example event for `queue` looks as following: ```$json { "@timestamp": "2020-06-25T10:15:10.955Z", - "dataset": { - "type": "metrics", - "name": "rabbitmq.queue", - "namespace": "default" - }, "rabbitmq": { - "node": { - "name": "rabbit@047b9c4733f5" - }, "queue": { "auto_delete": false, "state": "running", @@ -490,11 +566,6 @@ An example event for `queue` looks as following: "type": "rabbitmq", "address": "localhost:15672" }, - "stream": { - "dataset": "rabbitmq.queue", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" } @@ -522,6 +593,7 @@ An example event for `queue` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -557,3 +629,5 @@ An example event for `queue` looks as following: | rabbitmq.queue.name | The name of the queue with non-ASCII characters escaped as in C. | keyword | | rabbitmq.queue.state | The state of the queue. Normally 'running', but may be "{syncing, MsgCount}" if the queue is synchronising. Queues which are located on cluster nodes that are currently down will be shown with a status of 'down'. | keyword | | rabbitmq.vhost | Virtual host name with non-ASCII characters escaped as in C. | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | diff --git a/packages/rabbitmq/manifest.yml b/packages/rabbitmq/manifest.yml index 45525d2ced6..f649cb20024 100644 --- a/packages/rabbitmq/manifest.yml +++ b/packages/rabbitmq/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: rabbitmq title: RabbitMQ -version: 0.2.6 +version: 0.2.7 license: basic description: RabbitMQ Integration type: integration From 2cfc1a76d4a04a137145b05506448cc620a71117 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 2 Mar 2021 17:08:57 +0100 Subject: [PATCH 18/18] Use latest elastic-package --- go.mod | 3 +-- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 0df40738065..b58504bbfa8 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,8 @@ go 1.12 require ( github.com/blang/semver v3.5.1+incompatible - github.com/elastic/elastic-package v0.0.0-20210301151806-9253ee3cdcfe + github.com/elastic/elastic-package v0.0.0-20210302155343-7a8f99157a71 github.com/elastic/package-registry v0.17.0 - github.com/elastic/package-spec/code/go v0.0.0-20210302092944-28bc03fffa83 // indirect github.com/magefile/mage v1.11.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index 501065be143..660e15dc816 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/elastic/elastic-package v0.0.0-20210301151806-9253ee3cdcfe h1:zNv+fOiAXa2i9aqzxw+ht+ftfntSXLNJUlSd9oPUjwQ= -github.com/elastic/elastic-package v0.0.0-20210301151806-9253ee3cdcfe/go.mod h1:x4ojhiQp1YW2dADwv8QYyx9hNLuto4jarprEO8qUx7c= +github.com/elastic/elastic-package v0.0.0-20210302155343-7a8f99157a71 h1:uVs37M0pMJBtRZuiRbRsGxZvNjGdQc4anbyI1VFahuc= +github.com/elastic/elastic-package v0.0.0-20210302155343-7a8f99157a71/go.mod h1:NjAFCbU9bzFpHIq8bGmcM7ejGNuqJ2KgN8gFb3qcvsU= github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M= github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= @@ -94,7 +94,6 @@ github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6 h1:Ehbr7du4rSSEy github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= github.com/elastic/package-registry v0.17.0 h1:Gh7u3TlHA3GJh+C/OZ8Pf4EUrFxcCXMAe2kUCjAiYgQ= github.com/elastic/package-registry v0.17.0/go.mod h1:fMVt9ozLSPAIgYTDgV23IZrSoDKZma7VKpA4uSkfPts= -github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= github.com/elastic/package-spec/code/go v0.0.0-20210302092944-28bc03fffa83 h1:MBgnWdr/ygkPsD8cL6INTIjH8TpUmpMZ4RqjmMaJAvk= github.com/elastic/package-spec/code/go v0.0.0-20210302092944-28bc03fffa83/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=